build: Add download capability for external packages

The DPMA and g729a, silk, siren7 and siren14 codecs hosted at
http://downloads.digium.com/pub/telephony/ are now listed in the
"External" sections of the "Resource Modules" and "Codec Translators"
pages in menuselect.  Any that are selected will automatically be
downloaded and installed when "make install" is run.  Their LICENSE and
README (if avaialble) files will be installed to
ASTVARLIBDIR/documentation/thirdparty/<product_name>.

Example use with codecs:

The codecs/codecs.xml file is a menuselect style xml file that lists
the codecs to be included.  Their support levels are 'external', which
triggers the download and install, and defaultenabled is no.  Also
because codec_g729a is actually in a directory named codec_g729 on the
download server, the newly added 'member_data' element is used to
override the default of the directory name being the package name.  You
can use the 'directory_name' attribute to keep default base URL
(http://downloads.digium.com/pub/telephony/) but use the new directory,
or you use the 'remote_url' attribute to specify a full URL to the
download directory.  In this case, you must still follow the same
subdirectory naming conventions as that used for the packages located
at 'http://downloads.digium.com/pub/telephony'.

A new configure option '--with-externals-cache' was added and like
'--with-sounds-cache' it allows the installer to cache tarballs so
they're not downloaded every time.

To assist with the download and install process, each external package
now has a manifest.xml file that, among other things, contains a package
version and checksums for each file in the tarball.  The manifest is
saved to both the cache directory and ASTMODDIR and together with the
manifest.xml on the downloads site, tells the install scripts whether
a download and/or update is needed.

bash and xmlstarlet are required for downloader operation.  If they're
not installed, the external items in menuselect will be unavailable.

Change-Id: Id3dcf1289ffd3cb0bbd7dfab3cafbb87be60323a

4 files changed, 240 insertions, 1 deletions

diff --git a/build_tools/download_externals b/build_tools/download_externals
new file mode 100755
index 000000000..8d5872df7
--- /dev/null
+++ b/build_tools/download_externals
@@ -0,0 +1,180 @@
+#!/usr/bin/env bash
+
+if [[ ( ${BASH_VERSINFO[0]} == 4 && ${BASH_VERSINFO[1]} > 1 ) || ${BASH_VERSINFO[0]} > 4 ]] ; then
+	shopt -s compat41
+fi
+set -e
+
+ASTTOPDIR=${ASTTOPDIR:-.}
+
+module_name=$1
+
+if [[ -z ${module_name} ]] ; then
+	echo "You must supply a module name."
+	exit 64
+fi
+
+tmpdir=$(mktemp -d)
+if [[ -z "${tmpdir}" ]] ; then
+	echo "${module_name}: Unable to create temporary directory."
+	exit 1
+fi
+trap "rm -rf ${tmpdir}" EXIT
+
+sed -r -e "s/^([^ =]+)\s*=\s*(.*)$/\1=\"\2\"/g" ${ASTTOPDIR}/makeopts >${tmpdir}/makeopts
+source ${tmpdir}/makeopts
+if [[ -z "${ASTMODDIR}" ]] ; then
+	echo "${module_name}: Unable to parse ${ASTTOPDIR}/makeopts."
+	exit 1
+fi
+
+XMLSTARLET=${XMLSTARLET:-xmlstarlet}
+if [[ "${XMLSTARLET}" = ":" ]] ; then
+	echo "${module_name}: The externals downloader requires xmlstarlet to be installed."
+	exit 1
+fi
+
+cache_dir="${EXTERNALS_CACHE_DIR}"
+if [[ -z ${cache_dir} ]] ; then
+	cache_dir=${tmpdir}
+fi
+
+version=$(${ASTTOPDIR}/build_tools/make_version ${ASTTOPDIR})
+if [[ ! ${version} =~ ^(GIT-)?([^.-]+)[.-].* ]] ; then
+	echo "${module_name}: Couldn't parse version ${version}"
+	exit 1
+fi
+major_version=${BASH_REMATCH[2]}
+
+if [[ "${major_version}" == "master" ]] ; then
+	echo "${module_name}: External module downloading is not available in the 'master' git branch.  Please disable in menuselect and download manually."
+	exit 1
+fi
+
+major_version=${major_version}.0
+
+if [[ "${HOST_CPU}" = "x86_64" ]] ; then
+	host_bits=64
+else
+	host_bits=32
+fi
+
+remote_url=$(${XMLSTARLET} sel -t -v "/menu/category/member[@name = '${module_name}']/member_data/downloader/@remote_url" ${ASTTOPDIR}/menuselect-tree || :)
+if [[ -n "${remote_url}" ]] ; then
+	remote_url="${remote_url}/asterisk-${major_version}/x86-${host_bits}"
+else
+	directory_name=$(${XMLSTARLET} sel -t -v "/menu/category/member[@name = '${module_name}']/member_data/downloader/@directory_name" ${ASTTOPDIR}/menuselect-tree || :)
+	remote_url="http://downloads.digium.com/pub/telephony/${directory_name:-${module_name}}/asterisk-${major_version}/x86-${host_bits}"
+fi
+
+version_convert() {
+	local v=${1##*_}
+	if [[ ${v} =~ ([0-9]+)[.]([0-9]+)[.]([0-9]+) ]] ; then
+		v=$(( ${BASH_REMATCH[1]}<<18 | ${BASH_REMATCH[2]}<<9 | ${BASH_REMATCH[3]} ))
+	fi
+	echo ${v}
+}
+
+${WGET} -q -O ${tmpdir}/manifest.xml ${remote_url}/manifest.xml || {
+	echo "${module_name}: Unable to fetch ${remote_url}/manifest.xml"
+	exit 1
+}
+
+rpv=$(${XMLSTARLET} sel -t -v "/package/@version" ${tmpdir}/manifest.xml)
+rpvi=$(version_convert ${rpv})
+echo "${module_name}: Remote package version ${rpv} (${rpvi})"
+
+module_dir=${module_name}-${rpv}-x86_${host_bits}
+tarball=${module_dir}.tar.gz
+export need_install=0
+
+if [[ -f ${DESTDIR}${ASTMODDIR}/${module_name}.manifest.xml ]] ; then
+	package_arch=$(${XMLSTARLET} sel -t -v "/package/@arch" ${DESTDIR}${ASTMODDIR}/${module_name}.manifest.xml)
+	ipv=$(${XMLSTARLET} sel -t -v "/package/@version" ${DESTDIR}${ASTMODDIR}/${module_name}.manifest.xml)
+	ipvi=$(version_convert ${ipv})
+	ip_major=${ipv%_*}
+	echo "${module_name}: Installed package version ${ipv} (${ipvi})"
+	if [[ "${ip_major}" != "${major_version}" || "${package_arch}" != "x86_${host_bits}" ]] ; then
+		echo "${module_name}: The installed package is not for this version of Asterisk.  Reinstalling."
+		need_install=1
+	elif [[ ${rpvi} > ${ipvi} ]] ; then
+		echo "${module_name}: A newer package is available"
+		need_install=1
+	else
+		sums=$(${XMLSTARLET} sel -t -m "//file" -v "@md5sum" -n ${DESTDIR}${ASTMODDIR}/${module_name}.manifest.xml)
+		for sum in ${sums} ; do
+			install_path=$(${XMLSTARLET} sel -t -v "//file[@md5sum = '${sum}']/@install_path" ${DESTDIR}${ASTMODDIR}/${module_name}.manifest.xml )
+			f=${DESTDIR}$(eval echo ${install_path})
+			if [[ ! -f ${f} ]] ; then
+				echo Not found: ${f}
+				need_install=1
+			else
+				cs=$(md5sum ${f} | cut -b1-32)
+				if [[ "${cs}" !=  "${sum}" ]] ; then
+					echo Checksum mismatch: ${f}
+					need_install=1
+				fi
+			fi
+		done
+	fi
+else
+	need_install=1
+fi
+
+if [[ ${need_install} == 1 ]] ; then
+	if [[ ( -n "${ipvi}" ) && ${ipvi} > ${rpvi} ]] ; then
+		echo "${module_name}: Installed package is newer than that available for download."
+		exit 0
+	fi
+else
+	echo "${module_name} is up to date."
+	exit 0;
+fi
+
+need_download=1
+if [[ -f ${cache_dir}/${module_name}.manifest.xml ]] ; then
+	cpv=$(${XMLSTARLET} sel -t -v "/package/@version" ${cache_dir}/${module_name}.manifest.xml)
+	cpvi=$(version_convert ${cpv})
+	echo "${module_name}: Cached package version ${cpv} (${cpvi})"
+	if [[ ${cpvi} == ${rpvi} && ( -f ${cache_dir}/${tarball} ) ]] ; then
+		echo "${module_name}: Cached version is available."
+		need_download=0
+	fi
+fi
+
+if [[ ${need_download} = 1 ]] ; then
+	echo "${module_name}: Downloading ${remote_url}/${tarball}"
+	${WGET} -q -O ${cache_dir}/${tarball} ${remote_url}/${tarball} || {
+		echo "${module_name}: Unable to fetch ${remote_url}/${tarball}"
+		exit 1
+	}
+	cp ${tmpdir}/manifest.xml  ${cache_dir}/${module_name}.manifest.xml
+fi
+
+tar -xzf ${cache_dir}/${tarball} -C ${cache_dir}
+trap "rm -rf ${cache_dir}/${module_dir} ; rm -rf ${tmpdir}" EXIT
+
+echo "${module_name}: Installing."
+
+if [[ $EUID == 0 ]] ; then
+	install_params="--group=0 --owner=0"
+fi
+
+names=$(${XMLSTARLET} sel -t -m "//file" -v "@name" -n ${cache_dir}/${module_dir}/manifest.xml)
+for name in ${names} ; do
+	source_path=${cache_dir}/${module_dir}/${name}
+	install_path=$(${XMLSTARLET} sel -t -v "//file[@name = '${name}']/@install_path" ${cache_dir}/${module_dir}/manifest.xml)
+	install_path=${DESTDIR}$(eval echo ${install_path})
+	executable=$(${XMLSTARLET} sel -t -v "//file[@name = '${name}']/@executable" ${cache_dir}/${module_dir}/manifest.xml || :)
+	if [[ "${executable}" = "yes" ]] ; then
+		mode=0755
+	else
+		mode=0644
+	fi
+
+	${INSTALL} -Dp ${install_params} --mode=${mode} ${source_path} ${install_path}
+
+done
+${INSTALL} -Dp ${install_params} --mode=0644 ${cache_dir}/${module_dir}/manifest.xml ${DESTDIR}${ASTMODDIR}/${module_name}.manifest.xml
+
+echo "${module_name}: Installed."
diff --git a/build_tools/list_valid_installed_externals b/build_tools/list_valid_installed_externals
new file mode 100755
index 000000000..12aff3f95
--- /dev/null
+++ b/build_tools/list_valid_installed_externals
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+if [[ ( ${BASH_VERSINFO[0]} == 4 && ${BASH_VERSINFO[1]} > 1 ) || ${BASH_VERSINFO[0]} > 4 ]] ; then
+	shopt -s compat41
+fi
+set -e
+
+ASTTOPDIR=${ASTTOPDIR:-.}
+
+tmpdir=$(mktemp -d)
+if [[ -z "${tmpdir}" ]] ; then
+	echo "${module_name}: Unable to create temporary directory."
+	exit 1
+fi
+trap "rm -rf ${tmpdir}" EXIT
+
+sed -r -e "s/^([^ =]+)\s*=\s*(.*)$/\1=\"\2\"/g" ${ASTTOPDIR}/makeopts >${tmpdir}/makeopts
+source ${tmpdir}/makeopts
+if [[ -z "${ASTMODDIR}" ]] ; then
+	echo "${module_name}: Unable to parse ${ASTTOPDIR}/makeopts."
+	exit 1
+fi
+
+XMLSTARLET=${XMLSTARLET:-xmlstarlet}
+if [[ "${XMLSTARLET}" = ":" ]] ; then
+	echo "${module_name}: The externals downloader requires xmlstarlet to be installed."
+	exit 1
+fi
+
+version=$(${ASTTOPDIR}/build_tools/make_version ${ASTTOPDIR})
+if [[ ! ${version} =~ ^(GIT-)?([^.-]+)[.-].* ]] ; then
+	echo "${module_name}: Couldn't parse version ${version}"
+	exit 1
+fi
+major_version=${BASH_REMATCH[2]}.0
+
+if [[ "${HOST_CPU}" = "x86_64" ]] ; then
+	host_bits=64
+else
+	host_bits=32
+fi
+
+names=""
+for manifest in ${DESTDIR}${ASTMODDIR}/*.manifest.xml ; do
+	if [ ! -f "$manifest" ] ; then
+		break
+	fi
+	package_version=$(${XMLSTARLET} sel -t -v "/package/@version" ${manifest})
+	package_major_version=${package_version%_*}
+	package_arch=$(${XMLSTARLET} sel -t -v "/package/@arch" ${manifest})
+	if [[ "$package_major_version" = "$major_version" && "${package_arch}" = "x86_${host_bits}" ]] ; then
+		names+=$(${XMLSTARLET} sel -t -m "//file[@executable = 'yes']" -v "concat(@name, ' ')" ${manifest})
+	fi
+done
+echo $names
diff --git a/build_tools/make_version b/build_tools/make_version
index fd14a550a..3e0f4a0b0 100755
--- a/build_tools/make_version
+++ b/build_tools/make_version
@@ -89,11 +89,13 @@ elif [ -d ${1}/.git ]; then
     if [ -z ${GIT} ]; then
         GIT="git"
     fi
-
+	
     if ! command -v ${GIT} >/dev/null 2>&1; then
         echo "UNKNOWN__and_probably_unsupported"
         exit 1
     fi
+	cd ${1} 
+    
     # If the first log commit messages indicates that this is checked into
     # subversion, we'll just use the SVN- form of the revision.
     MODIFIED=""
diff --git a/build_tools/menuselect-deps.in b/build_tools/menuselect-deps.in
index a04440955..9b077680e 100644
--- a/build_tools/menuselect-deps.in
+++ b/build_tools/menuselect-deps.in
@@ -31,6 +31,8 @@ KQUEUE=@PBX_KQUEUE@
 LDAP=@PBX_LDAP@
 LIBEDIT=@PBX_LIBEDIT@
 LIBXML2=@PBX_LIBXML2@
+XMLSTARLET=@PBX_XMLSTARLET@
+BASH=@PBX_BASH@
 LTDL=@PBX_LTDL@
 LUA=@PBX_LUA@
 MISDN=@PBX_MISDN@