Split the sync stage into fetch+prep

* At the core of the issue is that multiple packages can depend on the
  same sources.  To that end, the sources themselves need to be
  represented independant of the packages which require them, as well as
  needed to be archived/maintained independant of the packages.  To do
  this we have split the fetch action out separate of the prep action,
  thus fetch actions are only related to the source archives, and the
  prep action is how a package extracts said archive for use by a
  specific package.

4 files changed, 318 insertions, 142 deletions

diff --git a/scripts/builder/build-fetch b/scripts/builder/build-fetch
new file mode 100755
index 0000000..b6e7986
--- /dev/null
+++ b/scripts/builder/build-fetch
@@ -0,0 +1,179 @@
+#!/usr/bin/env build
+
+build_fetch_clean()
+{
+	for arg in ${BUILD_FETCH_CLEAN}; do
+		if [ -d "${arg}" ]; then
+			rm -rf "${arg}"
+		elif [ -f "${arg}" ]; then
+			rm "${arg}"
+		fi
+	done
+}
+
+build_fetch_uri()
+{
+	echo -n "${1%%\?*}"
+}
+
+build_fetch_args()
+{
+	set -- $(echo "${1##$(build_fetch_uri "${1}")\?}"|sed -e 's/&/ /')
+	echo -n "${@}"
+}
+
+build_fetch_name()
+{
+	build_fetch_name_var="$(build_fetch_uri "${1}")"
+	build_fetch_name_var="${build_fetch_name_var##*/}"
+	build_fetch_name_var="${build_fetch_name_var%%\?*}"
+
+	build_fetch_name_complete="0"
+	case "${build_fetch_name_var}" in
+	(*.tar.*)		build_fetch_name_complete=1;;
+	(*.t[bgx]z)		build_fetch_name_complete=1;;
+	(*.tbz2)		build_fetch_name_complete=1;;
+	(*.tz)			build_fetch_name_complete=1;;
+	esac
+
+	if [ "${build_fetch_name_complete}" -eq "1" ]; then
+		echo -n "${build_fetch_name_var}"
+		unset build_fetch_name_var
+		return
+	fi
+
+	# The filename to archive has to be supplied on the SOURCES_URI, else
+	# we can't do anything about packages which share common sources.
+	build_fetch_name_var=
+	for arg in $(build_fetch_args "${1}"); do
+		case "${arg}" in
+		(archive=*) build_fetch_name_var="${arg##*=}";;
+		esac
+	done
+
+	if [ -z "${build_fetch_name_var}" ]; then
+		die "do not know how to store source from '${1}'"
+	fi
+
+	echo -n "${build_fetch_name_var}"
+	unset build_fetch_name_var
+}
+
+build_fetch_var()
+{
+	echo -n "$(build_fetch_name "${1}" | sed -e 's/[.-]/_/g')"
+}
+
+build_fetch_git()
+{
+	build_fetch_git_uri="$(build_fetch_uri "${1}")"
+	build_fetch_git_tag=
+	build_fetch_git_var=
+	for arg in $(build_fetch_args "${1}"); do
+		case "${arg}" in
+		(archive=*) build_fetch_git_var="${arg##*=}";;
+		(*)		 build_fetch_git_tag="${arg}";;
+		esac
+	done
+	if [ -z "${build_fetch_git_var}" ]; then
+		die "do not know how to store source from '${1}'"
+	fi
+
+	# We want to avoid avoid copying the repository and all of its history
+	# in order to perform a single build, unfortunately git does not give a
+	# clear-cut approach to checkout out a specific tag or the head of a
+	# specific branch, instead we have to jump through some hoops.
+	build_fetch_git_tmp="$(mktemp -d /tmp/builder_git.XXXXXX)"
+	build_fetch_git_dir="${build_fetch_git_var%%.t*}"
+	cd "${build_fetch_git_tmp}"
+	BUILD_FETCH_CLEAN="${BUILD_FETCH_CLEAN} ${build_fetch_git_tmp}"
+	trap build_fetch_clean EXIT
+	if ! git clone --depth 1 "${build_fetch_git_uri}" "${build_fetch_git_dir}"; then
+		die "failed to clone git source at '${build_fetch_git_uri}'"
+	fi
+	if [ "${build_fetch_git_tag}" != "${build_fetch_git_uri}" ]; then
+		if ! cd "${build_fetch_git_dir}"; then
+			die "failed to change working directory to '${build_fetch_git_dir}'"
+		fi
+
+		if ! git fetch --depth 1 "${build_fetch_git_uri}" "${build_fetch_git_tag}"; then
+			die "failed to fetch git branch/tag '${build_fetch_git_tag}'"
+		fi
+
+		if ! git checkout "${build_fetch_git_tag}"; then
+			die "failed to checkout git branch/tag '${build_fetch_git_tag}'"
+		fi
+	fi
+
+	cd "${build_fetch_git_tmp}"
+	tar caf "${build_fetch_git_var}" "${build_fetch_git_dir}"
+	mv "${build_fetch_git_var}" "${BUILDER_SRCDIR}/${build_fetch_git_var}"
+
+	if [ -d "${build_fetch_git_tmp}" ]; then
+		rm -rf "${build_fetch_git_tmp}"
+	fi
+
+	unset build_fetch_git_tmp
+	unset build_fetch_git_var
+	unset build_fetch_git_dir
+	unset build_fetch_git_uri
+	unset build_fetch_git_tag
+}
+
+build_fetch_http()
+{
+	build_fetch_http_file="$(build_fetch_name "${1}")"
+
+	if [ ! -d "${BUILDER_SRCDIR}" ]; then
+		mkdir -p "${BUILDER_SRCDIR}"
+	fi
+	if [ ! -f "${BUILDER_SRCDIR}/${build_fetch_http_file}" ]; then
+		build_fetch_http_tmp="$(mktemp /tmp/builder_${NAME}.XXXXXX)"
+		if command -v "wget" > /dev/null 2>&1; then
+			wget -O "${build_fetch_http_tmp}" "${1}" || return 1
+		fi
+
+		if mv "${build_fetch_http_tmp}" "${BUILDER_SRCDIR}/${build_fetch_http_file}"; then
+			exit 1
+		fi
+		unset build_fetch_http_tmp
+	fi
+
+	unset build_fetch_http_file
+}
+
+FETCH_ACTION="fetch"
+while [ "$#" -gt "0" ]; do
+	case "${1}" in
+	(-N|-name|--name)
+		FETCH_ACTION="name"
+		shift
+		;;
+	(-V|-var|--var|--variable)
+		FETCH_ACTION="var"
+		shift
+		;;
+	(-*)	die "unknown fetch action '${1}'";;
+	(*)	break;;
+	esac
+done
+
+case "${FETCH_ACTION}" in
+(name)	build_fetch_name "${1}"; exit $?;;
+(var)	build_fetch_var "${1}"; exit $?;;
+(fetch)	break;;
+(*)	die "unknown fetch action '${FETCH_ACTION}'";;
+esac
+
+echo "fetch: ${1}"
+
+# FIXME this stuff needs a lot of work
+case "${1}" in
+(git://*)	build_fetch_git "${1}";;
+(http://*)	build_fetch_http "${1}";;
+(ftp://*)	build_fetch_http "${1}";;
+# Everything else is death
+(*)		die "do not know how to handle '${1}'"
+esac
+
+# vim: filetype=sh
diff --git a/scripts/builder/build-makedeps b/scripts/builder/build-makedeps
index d9ac844..c12ed17 100755
--- a/scripts/builder/build-makedeps
+++ b/scripts/builder/build-makedeps
@@ -3,7 +3,8 @@ cat >"${BUILDER_MAKEFILE}" <<EOF
 ##
 # Some generic catchall rules
 all: packages_archive
-all_sync: packages_sync
+all_fetch: packages_fetch
+all_prep: packages_prep
 all_clean: packages_clean sysroot_clean
 all_distclean: packages_distclean sysroot_clean artifacts_clean tooldir_clean
 all_compile: packages_compile
@@ -22,7 +23,8 @@ artifacts_clean:
 EOF
 
 PACKAGES=
-PACKAGES_SYNC=
+PACKAGES_PREP=
+PACKAGES_FETCH=
 PACKAGES_CLEAN=
 PACKAGES_COMPILE=
 PACKAGES_ARCHIVE=
@@ -34,13 +36,43 @@ for package in $(cd "${BUILDER_PKGDIR}" && echo */*); do
 	fi
 
 	package_make="$(echo "${package}"|tr '/-' '__')"
-	package_archive="${BUILDER_ATFDIR}/${NAME}-${VERSION}.${ARCHIVE_FORMAT}"
-	package_rulesfile="${RULESFILE}"
-	package_logdir="${L}"
 	package_install="${W}/.installed"
 	package_compile="${W}/.compiled"
-	package_sync="${W}/.synced"
-	# FIXME we need to deal with package_source as well
+	package_prep="${W}/.prepped"
+	package_logdir="${L}"
+
+	if [ -z "${TOOLCHAIN}" ]; then
+		package_archive="${BUILDER_ATFDIR}/${CHOST}/${NAME}-${VERSION}.${ARCHIVE_FORMAT}"
+	else
+		package_archive="${BUILDER_ATFDIR}/${CBUILD}/${NAME}-${VERSION}.${ARCHIVE_FORMAT}"
+	fi
+
+	package_sources="${RULESFILE}"
+	for patch in ${PATCHES}; do
+		patch="${F}/${NAME}-${VERSION}-${patch}.patch"
+		if [ ! -f "${patch}" ]; then
+			die "patch does not exist '${patch}'"
+		fi
+		package_sources="${package_sources} ${patch}"
+	done
+
+	# FIXME there can be strange characters in a URI .. this might not be
+	# the best approach in the long term.
+	for url in ${SOURCE_URI}; do
+		case "${url}" in
+		# Do not translate local paths into archives in BUILDER_SRCDIR
+		(file://*|/*)	packages_sources="${package_sources} ${url##file://}";;
+
+		# Assume anything else with :// in the name is remote
+		(*://*)		pkg_src="$(build-fetch --name "${url}")"
+				if [ $? -ne 0 ]; then
+					exit 1
+				fi
+				package_sources="${package_sources} ${BUILDER_SRCDIR}/${pkg_src}";;
+		# Junk?
+		(*)		die "do not know how to handle url '${url}'";;
+		esac
+	done
 
 	# This loop can end up being fairly costly if we have to fire up
 	# build-query, particularly when dealing with a large number of
@@ -70,7 +102,7 @@ ${package_make}: ${package_archive}
 ${package_make}_install: ${package_install}
 ${package_make}_archive: ${package_archive}
 ${package_make}_compile: ${package_compile}
-${package_make}_sync: ${package_sync}
+${package_make}_fetch: ${package_sources}
 ${package_make}_makedeps:
 ${package_make}_clean:
 	@build-clean ${CATEGORY}/${NAME}
@@ -84,24 +116,54 @@ ${package_archive}: ${package_compile}
 		echo "logfile: '${L}/archive.log'" >&2 ;\
 		exit 1; \
 	fi
-${package_compile}: ${package_sync}${package_deps}
+${package_compile}: ${package_prep}${package_deps}
 	@if ! build-compile ${CATEGORY}/${NAME}; then \
 		echo "error: failed to compile package '${CATEGORY}/${NAME}'" >&2 ;\
 		echo "logfile: '${L}/compile.log'" >&2 ;\
 		exit 1 ;\
 	fi
-${package_sync}: ${package_rule}
-	@if ! build-sync ${CATEGORY}/${NAME}; then \
-		echo "error: failed to syncronize package '${CATEGORY}/${NAME}'" >&2 ;\
-		echo "logfile: '${L}/sync.log'" >&2 ;\
+${package_prep}: ${package_sources}
+	@if ! build-prep ${CATEGORY}/${NAME}; then \
+		echo "error: failed to prepare package '${CATEGORY}/${NAME}'" >&2 ;\
+		echo "logfile: '${L}/prep.log'" >&2 ;\
 		exit 1 ;\
 	fi
-
 EOF
 
+	##
+	# This is a bit of a fun late-injection of the source archive for a
+	# package.  The core problem is that multiple packages may depend on
+	# the same sources, so we set up a separate rule for the source
+	# archive when processing the package, but only if an existing entry
+	# for that source does not exist.  We use the source name (as opposed
+	# to the package name) to track if the package already has a rule.  The
+	# whole thing really translates into something like
+	# foo_1_1_3_tar_bz2="http://some/path/to/foo-1.1.3.tar.bz2"
+	# All forms of URL translation and variable translation are done for us
+	# by fetch so that makedeps doesn't have any specific expectations on
+	# what the variable name should be.
+
+	for url in ${SOURCE_URI}; do
+		case "${url}" in
+		# Do not translate local paths into archives in BUILDER_SRCDIR
+		(file://*|/*)	echo "${url##file://}:";;
+
+		# Assume anything else with :// in the name is remote
+		(*://*)
+			var="$(build-fetch --var "${url}")"
+			if [ -z "$(eval echo -n "\$${var}")" ]; then
+				eval "${var}='${url}'"
+				echo "${BUILDER_SRCDIR}/$(build-fetch --name "${url}"):"
+				echo "\t@build-fetch \"${url}\""
+			fi
+			;;
+		esac
+	done
+
 	if [ "${CATEGORY}" = "${PROJECT_NAME}" ]; then
 		PACKAGES="${PACKAGES} ${package_make}"
-		PACKAGES_SYNC="${PACKAGES_SYNC} ${package_sync}"
+		PACKAGES_FETCH="${PACKAGES_FETCH} ${package_fetch}"
+		PACKAGES_PREP="${PACKAGES_PREP} ${package_fetch}"
 		PACKAGES_COMPILE="${PACKAGES_COMPILE} ${package_compile}"
 		PACKAGES_ARCHIVE="${PACKAGES_ARCHIVE} ${package_archive}"
 		PACKAGES_INSTALL="${PACKAGES_INSTALL} ${package_install}"
@@ -112,7 +174,8 @@ EOF
 done >> "${BUILDER_MAKEFILE}"
 
 cat<<EOF >> "${BUILDER_MAKEFILE}"
-packages_sync: ${PACKAGES_SYNC}
+packages_fetch: ${PACKAGES_FETCH}
+packages_prep: ${PACKAGES_PREP}
 packages_clean: ${PACKAGES_CLEAN}
 packages_compile: ${PACKAGES_COMPILE}
 packages_archive: ${PACKAGES_ARCHIVE}
diff --git a/scripts/builder/build-prep b/scripts/builder/build-prep
new file mode 100755
index 0000000..9fe8def
--- /dev/null
+++ b/scripts/builder/build-prep
@@ -0,0 +1,60 @@
+#!/usr/bin/env build
+
+echo "prepping: ${1}"
+
+eval $(build-query --environ "${1}")
+
+if [ ! -d "${L}" ]; then
+	mkdir -p "${L}"
+fi
+
+if [ -f "${L}/prep.log" ]; then
+	rm "${L}/prep.log"
+fi
+touch "${L}/prep.log"
+
+for dir in "${D}" "${S}"; do
+	if [ -d "${dir}" ]; then
+		mv "${dir}" "${dir}.old"
+		rm -rf "${dir}.old" &
+	fi
+	mkdir -p "${dir}"
+done
+
+wait
+
+if [ -d "${BUILDER_PKGDIR}/${NAME}/source" ]; then
+	if [ "$(command -v rsync)" ]; then
+		if ! rsync -rav --delete "${BUILDER_PKGDIR}/${NAME}/source/" "${S}"; then
+			die "failed to sync source to '${S}'"
+		fi
+	else
+		if ! cp -vadpR "${BUILDER_PKGDIR}/${NAME}/source" "${S}"; then
+			die "failed to copy source to '${S}'"
+		fi
+	fi >> "${L}/prep.log" 2>&1
+else
+	# FIXME this stuff needs a lot of work
+	for url in ${SOURCE_URI}; do
+		file="$(build-fetch --name "${url}")"
+		if [ ! -f "${BUILDER_SRCDIR}/${file}" ]; then
+			die "source does not exist '${file}'"
+		fi
+
+		tar xavf "${BUILDER_SRCDIR}/${file}" -C "${W}" >> "${L}/prep.log" 2>&1
+	done
+fi
+
+# FIXME wrap up the patch command with something more functional
+if [ ! -z "${PATCHES}" ]; then
+	for patch in ${PATCHES}; do
+		echo "${NAME}: applying patch '${patch}'" | tee -a "${L}/prep.log"
+		if ! patch -l -t -d "${S}" -p1 < "${F}/${NAME}-${VERSION}-${patch}.patch"; then
+			exit 1
+		fi >> "${L}/prep.log" 2>&1
+	done
+fi
+
+date --utc > "${W}/.prepped"
+
+# vim: filetype=sh
diff --git a/scripts/builder/build-sync b/scripts/builder/build-sync
deleted file mode 100755
index c9cceba..0000000
--- a/scripts/builder/build-sync
+++ /dev/null
@@ -1,126 +0,0 @@
-#!/usr/bin/env build
-
-echo "syncing: ${1}"
-
-eval $(build query --environ "${1}")
-
-build_sync_git()
-{
-	GIT_URI="${1%%\?*}"
-	GIT_TAG="${1##*\?}"
-
-	# We want to avoid avoid copying the repository and all of it's history
-	# in order to perform a single build, unfortunately git does not give a
-	# clear-cut approach to checkout out a specific tag or the head of a
-	# specific branch, instead we have to jump through some hoops.
-	if [ ! -d "${S}/.git" ]; then
-		[ -d "${S}" ] && rm -rf "${S}"
-		git clone --depth 1 "${GIT_URI}" "${S}" || die "failed to clone git source at '${GIT_URI}'"
-	fi
-	if [ "${GIT_TAG}" != "${GIT_URI}" ]; then
-		cd "${S}" || die "failed to change working directory to '${S}'"
-		git fetch --depth 1 "${GIT_URI}" "${GIT_TAG}" || die "failed to fetch git branch/tag '${GIT_TAG}'"
-		git checkout "${GIT_TAG}" || die "failed to checkout git branch/tag '${GIT_TAG}'"
-	fi
-}
-
-build_sync_svn()
-{
-	if [ ! -d "${S}/.svn" ]; then
-		[ -d "${S}" ] && rm -rf "${S}"
-		svn checkout "${1}" "${S}" || die "failed to checkout svn source at '${1}'"
-	fi
-}
-
-build_sync_http()
-{
-	HTTP_FILE="${1##*/}"
-	HTTP_FILE="${HTTP_FILE%%\?*}"
-
-	if [ ! -d "${BUILDER_SRCDIR}" ]; then
-		mkdir -p "${BUILDER_SRCDIR}"
-	fi
-	if [ ! -f "${BUILDER_SRCDIR}/${HTTP_FILE}" ]; then
-		HTTP_TMP="$(mktemp /tmp/builder_${NAME}.XXXXXX)"
-		if command -v "wget" > /dev/null 2>&1; then
-			wget -O "${HTTP_TMP}" "${1}" || return 1
-		fi
-
-		mv "${HTTP_TMP}" "${BUILDER_SRCDIR}/${HTTP_FILE}" || return 1
-	fi
-
-	case "${HTTP_FILE}" in
-	(*.tar.bz|*.tbz|*.tar.bz2|*.tbz2)
-		bzip2 -dc "${BUILDER_SRCDIR}/${HTTP_FILE}" | tar x -C "${S}";;
-	(*.tar.[Zz]|*.tz)
-		gzip -dc "${BUILDER_SRCDIR}/${HTTP_FILE}" | tar x -C "${S}";;
-	(*.tar.gz|*.tgz)
-		gzip -dc "${BUILDER_SRCDIR}/${HTTP_FILE}" | tar x -C "${S}";;
-	(*)	die "do not know how to extract '${HTTP_FILE}'";;
-	esac
-
-	if [ -d "${S}/${NAME}-${VERSION}" ]; then
-		(cd "${S}" && find "${NAME}-${VERSION}" -maxdepth 1 \
-			! -name "${NAME}-${VERSION}" -a -print0 \
-			| xargs -0i mv -f "${S}/{}" "${S}/")
-		rmdir "${S}/${NAME}-${VERSION}"
-	fi
-}
-
-if [ ! -d "${L}" ]; then
-	mkdir -p "${L}"
-fi
-
-if [ -f "${L}/sync.log" ]; then
-	rm "${L}/sync.log"
-fi
-touch "${L}/sync.log"
-
-for dir in "${D}" "${S}"; do
-	if [ -d "${dir}" ]; then
-		echo "${NAME}: prepping '${dir}' for sync" >> "${L}/sync.log"
-		mv "${dir}" "${dir}.old"
-		rm -rf "${dir}.old" &
-	fi
-	mkdir -p "${dir}"
-done
-
-wait
-
-if [ -d "${BUILDER_PKGDIR}/${NAME}/source" ]; then
-	if [ "$(command -v rsync)" ]; then
-		if ! rsync -rav --delete "${BUILDER_PKGDIR}/${NAME}/source/" "${S}"; then
-			die "failed to sync source to '${S}'"
-		fi
-	else
-		if ! cp -vadpR "${BUILDER_PKGDIR}/${NAME}/source" "${S}"; then
-			die "failed to copy source to '${S}'"
-		fi
-	fi >> "${L}/sync.log" 2>&1
-else
-	# FIXME this stuff needs a lot of work
-	for url in ${SOURCE_URI}; do
-		case "${url}" in
-		(svn://*)	build_sync_svn "${url}";;
-		(git://*)	build_sync_git "${url}";;
-		(http://*)	build_sync_http "${url}";;
-		(ftp://*)	build_sync_http "${url}";;
-		# Everything else is death
-		(*)		die "do not know how to handle '${url}'"
-		esac >> "${L}/sync.log" 2>&1
-	done
-fi
-
-# FIXME wrap up the patch command with something more functional
-if [ ! -z "${PATCHES}" ]; then
-	for patch in ${PATCHES}; do
-		echo "${NAME}: applying patch '${patch}'" | tee -a "${L}/sync.log"
-		if ! patch -l -t -d "${S}" -p1 < "${F}/${NAME}-${VERSION}-${patch}.patch"; then
-			exit 1
-		fi >> "${L}/sync.log" 2>&1
-	done
-fi
-
-date --utc > "${W}/.synced"
-
-# vim: filetype=sh