xbps-src: refactor build logic into its own script (build.sh).

The build.sh script is now responsible to handle all the logic to build
a source package and its subpackages, as well as all its required
build dependencies. Thanks to this and subshells, dependencies are now
built into its own child process, creating a process tree that can go
nested as long as your system allows forking and has enough memory :-)

This fixes some issues that have been while building pkgs that have lots
of nested dependencies.

common/xbps-src/libexec/build.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+#
+# vim: set ts=4 sw=4 et:
+#
+# Passed arguments:
+#   $1 - current pkgname to build [REQUIRED]
+#   $2 - target pkgname (origin) to build [REQUIRED]
+#   $3 - xbps target [REQUIRED]
+#   $4 - cross target [OPTIONAL]
+
+if [ $# -lt 3 -o $# -gt 4 ]; then
+    echo "$(basename $0): invalid number of arguments: pkgname targetpkg target [cross-target]"
+    exit 1
+fi
+
+readonly PKGNAME="$1"
+readonly TARGET_PKG="$2"
+readonly TARGET="$3"
+readonly XBPS_CROSS_BUILD="$4"
+
+for f in $XBPS_SHUTILSDIR/*.sh; do
+    . $f
+done
+
+setup_pkg "$PKGNAME" $XBPS_CROSS_BUILD
+show_pkg_build_options
+check_pkg_arch $XBPS_CROSS_BUILD
+install_cross_pkg $XBPS_CROSS_BUILD
+
+# Install dependencies from binary packages
+if [ -z "$XBPS_SKIP_DEPS" ]; then
+    install_pkg_deps $PKGNAME $TARGET_PKG $TARGET $XBPS_CROSS_BUILD || exit 1
+fi
+
+# Fetch distfiles after installing required dependencies,
+# because some of them might be required for do_fetch().
+$XBPS_LIBEXECDIR/xbps-src-dofetch.sh $PKGNAME $XBPS_CROSS_BUILD || exit 1
+[ "$TARGET" = "fetch" ] && exit 0
+
+# Fetch, extract, build and install into the destination directory.
+$XBPS_LIBEXECDIR/xbps-src-doextract.sh $PKGNAME $XBPS_CROSS_BUILD || exit 1
+[ "$TARGET" = "extract" ] && exit 0
+
+# Run configure phase
+$XBPS_LIBEXECDIR/xbps-src-doconfigure.sh $PKGNAME $XBPS_CROSS_BUILD || exit 1
+[ "$TARGET" = "configure" ] && exit 0
+
+# Run build phase
+$XBPS_LIBEXECDIR/xbps-src-dobuild.sh $PKGNAME $XBPS_CROSS_BUILD || exit 1
+[ "$TARGET" = "build" ] && exit 0
+
+# Install pkgs into destdir.
+$XBPS_LIBEXECDIR/xbps-src-doinstall.sh $PKGNAME $XBPS_CROSS_BUILD || exit 1
+
+for subpkg in ${subpackages} ${sourcepkg}; do
+    $XBPS_LIBEXECDIR/xbps-src-doinstall.sh $subpkg $XBPS_CROSS_BUILD || exit 1
+done
+for subpkg in ${subpackages} ${sourcepkg}; do
+    $XBPS_LIBEXECDIR/xbps-src-prepkg.sh $subpkg $XBPS_CROSS_BUILD || exit 1
+done
+
+for subpkg in ${subpackages} ${sourcepkg}; do
+    if [ "$PKGNAME" = "${subpkg}" -a "$TARGET" = "install" ]; then
+        exit 0
+    fi
+done
+
+# If install went ok generate the binpkgs.
+for subpkg in ${subpackages} ${sourcepkg}; do
+    $XBPS_LIBEXECDIR/xbps-src-dopkg.sh $subpkg "$XBPS_REPOSITORY" "$XBPS_CROSS_BUILD" || exit 1
+done
+
+# pkg cleanup
+if declare -f do_clean >/dev/null; then
+    run_func do_clean
+fi
+
+if [ -z "$XBPS_KEEP_ALL" ]; then
+    remove_pkg_autodeps
+    remove_pkg_wrksrc
+    remove_pkg $XBPS_CROSS_BUILD
+    remove_pkg_statedir
+fi
+
+# If base-chroot not installed, install "base-files" into masterdir
+# from local repository; this is the only pkg required to be able to build
+# the bootstrap pkgs from scratch.
+if [ -z "$CHROOT_READY" -a "$PKGNAME" = "base-files" ]; then
+    msg_normal "Installing $PKGNAME into masterdir...\n"
+    _log=$(mktemp --tmpdir || exit 1)
+    if [ -n "$XBPS_BUILD_FORCEMODE" ]; then
+        _flags="-f"
+    fi
+    $XBPS_INSTALL_CMD ${_flags} -y $PKGNAME >${_log} 2>&1
+    if [ $? -ne 0 ]; then
+        msg_red "Failed to install $PKGNAME into masterdir, see below for errors:\n"
+        cat ${_log}
+        rm -f ${_log}
+        msg_error "Cannot continue!"
+    fi
+    rm -f ${_log}
+fi
+
+exit 0

common/xbps-src/libexec/xbps-src-dobuild.sh

@@ -24,11 +24,6 @@ for f in $XBPS_COMMONDIR/environment/build/*.sh; do
     source_file "$f"
 done
 
-if [ -z $pkgname -o -z $version ]; then
-    msg_error "$1: pkgname/version not set in pkg template!\n"
-    exit 1
-fi
-
 XBPS_BUILD_DONE="${XBPS_STATEDIR}/${sourcepkg}_${XBPS_CROSS_BUILD}_build_done"
 XBPS_PRE_BUILD_DONE="${XBPS_STATEDIR}/${sourcepkg}_${XBPS_CROSS_BUILD}_pre_build_done"
 XBPS_POST_BUILD_DONE="${XBPS_STATEDIR}/${sourcepkg}_${XBPS_CROSS_BUILD}_post_build_done"
@@ -68,7 +63,6 @@ else
     fi
 fi
 
-touch -f $XBPS_BUILD_DONE
 
 # Run post_build()
 if [ ! -f $XBPS_POST_BUILD_DONE ]; then
@@ -80,4 +74,6 @@ fi
 
 run_pkg_hooks post-build
 
+touch -f $XBPS_BUILD_DONE
+
 exit 0

common/xbps-src/libexec/xbps-src-doconfigure.sh

@@ -63,8 +63,6 @@ else
     fi
 fi
 
-touch -f $XBPS_CONFIGURE_DONE
-
 # Run post_configure()
 if [ ! -f $XBPS_POSTCONFIGURE_DONE ]; then
     if declare -f post_configure >/dev/null; then
@@ -75,4 +73,6 @@ fi
 
 run_pkg_hooks post-configure
 
+touch -f $XBPS_CONFIGURE_DONE
+
 exit 0

common/xbps-src/libexec/xbps-src-doextract.sh

@@ -59,7 +59,6 @@ else
     fi
 fi
 
-touch -f $XBPS_EXTRACT_DONE
 
 [ -d $wrksrc ] && cd $wrksrc
 
@@ -71,4 +70,6 @@ fi
 # Run post-extract hooks
 run_pkg_hooks post-extract
 
+touch -f $XBPS_EXTRACT_DONE
+
 exit 0

common/xbps-src/libexec/xbps-src-dofetch.sh

@@ -43,7 +43,6 @@ if declare -f do_fetch >/dev/null; then
     cd ${XBPS_BUILDDIR}
     [ -n "$build_wrksrc" ] && mkdir -p "$wrksrc"
     run_func do_fetch
-    touch -f $XBPS_FETCH_DONE
 else
     # Run do-fetch hooks.
     run_pkg_hooks "do-fetch"
@@ -57,4 +56,6 @@ fi
 # Run post-fetch hooks.
 run_pkg_hooks post-fetch
 
+touch -f $XBPS_FETCH_DONE
+
 exit 0

common/xbps-src/libexec/xbps-src-prepkg.sh

@@ -45,6 +45,7 @@ fi
 source_file $XBPS_COMMONDIR/environment/build-style/${build_style}.sh
 setup_pkg_depends $pkgname
 run_pkg_hooks pre-pkg
+
 touch -f $XBPS_PREPKG_DONE
 
 exit 0

common/xbps-src/shutils/build_dependencies.sh

@@ -190,22 +190,26 @@ check_installed_pkg() {
 # Installs all dependencies required by a package.
 #
 install_pkg_deps() {
-    local pkg="$1" cross="$2" rval _realpkg curpkgdepname pkgn iver _props _exact
+    local pkg="$1" targetpkg="$2" target="$3" cross="$4"
+    local rval _realpkg curpkgdepname pkgn iver _props _exact
     local i j found rundep checkver
 
-    local -a host_binpkg_deps binpkg_deps
+    local -a host_binpkg_deps binpkg_deps host_missing_deps missing_deps
-    local -a host_missing_deps missing_deps
 
     [ -z "$pkgname" ] && return 2
 
     setup_pkg_depends
 
+    if [ "$pkg" != "$targetpkg" ]; then
+        msg_normal "$pkgver: building (dependency of $targetpkg) ...\n"
+    else
+        msg_normal "$pkgver: building ...\n"
+    fi
+
     if [ -z "$build_depends" -a -z "$host_build_depends" -a -z "$run_depends" ]; then
         return 0
     fi
 
-    msg_normal "$pkgver: required dependencies:\n"
-
     #
     # Host build dependencies.
     #
@@ -335,37 +339,38 @@ install_pkg_deps() {
 
     # Host missing dependencies, build from srcpkgs.
     for i in ${host_missing_deps[@]}; do
+        # packages not found in repos, install from source.
+        (
         curpkgdepname=$($XBPS_UHELPER_CMD getpkgdepname "$i")
         setup_pkg $curpkgdepname
         ${XBPS_UHELPER_CMD} pkgmatch "$pkgver" "$i"
         if [ $? -eq 0 ]; then
-            setup_pkg $XBPS_TARGET_PKG
+            setup_pkg $pkg
             msg_error "$pkgver: required host dependency '$i' cannot be resolved!\n"
         fi
-        install_pkg full
+        exec env XBPS_BINPKG_EXISTS=1 $XBPS_LIBEXECDIR/build.sh $sourcepkg $pkg $target || exit 1
-        setup_pkg $XBPS_TARGET_PKG $XBPS_CROSS_BUILD
+        ) || exit 1
-        install_pkg_deps $sourcepkg $XBPS_CROSS_BUILD
+        host_binpkg_deps+=("$i")
     done
 
     # Target missing dependencies, build from srcpkgs.
     for i in ${missing_deps[@]}; do
         # packages not found in repos, install from source.
+        (
         curpkgdepname=$($XBPS_UHELPER_CMD getpkgdepname "$i")
         setup_pkg $curpkgdepname $cross
-        # Check if version in srcpkg satisfied required dependency,
-        # and bail out if doesn't.
         $XBPS_UHELPER_CMD pkgmatch "$pkgver" "$i"
         if [ $? -eq 0 ]; then
-            setup_pkg $XBPS_TARGET_PKG $cross
+            setup_pkg $pkg $cross
             msg_error "$pkgver: required target dependency '$i' cannot be resolved!\n"
         fi
-        install_pkg full $cross
+        exec env XBPS_BINPKG_EXISTS=1 $XBPS_LIBEXECDIR/build.sh $sourcepkg $pkg $target $cross || exit 1
-        setup_pkg $XBPS_TARGET_PKG $XBPS_CROSS_BUILD
+        ) || exit 1
-        install_pkg_deps $sourcepkg $XBPS_CROSS_BUILD
+        binpkg_deps+=("$i")
     done
 
-    if [ "$TARGETPKG_PKGDEPS_DONE" ]; then
+    if [ "$pkg" != "$targetpkg" ]; then
-        return 0
+        msg_normal "$pkg: building (dependency of $targetpkg) ...\n"
     fi
 
     for i in ${host_binpkg_deps[@]}; do
@@ -374,14 +379,7 @@ install_pkg_deps() {
     done
 
     for i in ${binpkg_deps[@]}; do
-        if [ -n "$CHROOT_READY" -a "$build_style" = "meta" ]; then
-            continue
-        fi
         msg_normal "$pkgver: installing target dependency '$i' ...\n"
         install_pkg_from_repos "$i" $cross
     done
-
-    if [ "$XBPS_TARGET_PKG" = "$sourcepkg" ]; then
-        TARGETPKG_PKGDEPS_DONE=1
-    fi
 }

common/xbps-src/shutils/bulk.sh

@@ -68,14 +68,12 @@ bulk_update() {
     done
     echo
     for f in ${pkgs}; do
-        BEGIN_INSTALL=1
-        XBPS_TARGET_PKG="$f"
         read_pkg
         msg_normal "xbps-src: building ${pkgver} ...\n"
         if [ -n "$CHROOT_READY" -a -z "$IN_CHROOT" ]; then
             chroot_handler pkg $XBPS_TARGET_PKG
         else
-            install_pkg pkg $XBPS_CROSS_BUILD
+            $XBPS_LIBEXECDIR/build.sh $f $f pkg $XBPS_CROSS_BUILD || return 1
         fi
         if [ $? -ne 0 ]; then
             msg_error "xbps-src: failed to build $pkgver pkg!\n"
@@ -84,6 +82,6 @@ bulk_update() {
     if [ -n "$pkgs" -a -n "$args" ]; then
         echo
         msg_normal "xbps-src: updating your system, confirm to proceed...\n"
-        ${XBPS_SUCMD} "xbps-install --repository=$XBPS_REPOSITORY --repository=$XBPS_REPOSITORY/nonfree -u ${pkgs}"
+        ${XBPS_SUCMD} "xbps-install --repository=$XBPS_REPOSITORY --repository=$XBPS_REPOSITORY/nonfree -u ${pkgs}" || return 1
     fi
 }

common/xbps-src/shutils/pkgtarget.sh

@@ -24,106 +24,6 @@ check_pkg_arch() {
     fi
 }
 
-install_pkg() {
-    local target="$1" cross="$2" lrepo subpkg opkg
-
-    [ -z "$pkgname" ] && return 1
-
-    show_pkg_build_options
-    check_pkg_arch $cross
-    install_cross_pkg $cross
-
-    if [ -z "$XBPS_SKIP_DEPS" ]; then
-        install_pkg_deps $sourcepkg $cross || return 1
-        if [ "$TARGETPKG_PKGDEPS_DONE" ]; then
-            setup_pkg $XBPS_TARGET_PKG $cross
-            unset TARGETPKG_PKGDEPS_DONE
-            install_cross_pkg $cross
-        fi
-    fi
-
-    # Fetch distfiles after installing required dependencies,
-    # because some of them might be required for do_fetch().
-    $XBPS_LIBEXECDIR/xbps-src-dofetch.sh $sourcepkg $cross || exit 1
-    [ "$target" = "fetch" ] && return 0
-
-    # Fetch, extract, build and install into the destination directory.
-    $XBPS_LIBEXECDIR/xbps-src-doextract.sh $sourcepkg $cross || exit 1
-    [ "$target" = "extract" ] && return 0
-
-    # Run configure phase
-    $XBPS_LIBEXECDIR/xbps-src-doconfigure.sh $sourcepkg $cross || exit 1
-    [ "$target" = "configure" ] && return 0
-
-    # Run build phase
-    $XBPS_LIBEXECDIR/xbps-src-dobuild.sh $sourcepkg $cross || exit 1
-    [ "$target" = "build" ] && return 0
-
-    # Install pkgs into destdir.
-    $XBPS_LIBEXECDIR/xbps-src-doinstall.sh $sourcepkg $cross || exit 1
-
-    for subpkg in ${subpackages} ${sourcepkg}; do
-        $XBPS_LIBEXECDIR/xbps-src-doinstall.sh $subpkg $cross || exit 1
-    done
-    for subpkg in ${subpackages} ${sourcepkg}; do
-        $XBPS_LIBEXECDIR/xbps-src-prepkg.sh $subpkg $cross || exit 1
-    done
-
-    for subpkg in ${subpackages} ${sourcepkg}; do
-        if [ "$XBPS_TARGET_PKG" = "${subpkg}" -a "$target" = "install" ]; then
-            return 0
-        fi
-    done
-
-    # If install went ok generate the binpkgs.
-    for subpkg in ${subpackages} ${sourcepkg}; do
-        $XBPS_LIBEXECDIR/xbps-src-dopkg.sh $subpkg "$XBPS_REPOSITORY" "$cross" || exit 1
-    done
-
-    # pkg cleanup
-    if declare -f do_clean >/dev/null; then
-        run_func do_clean
-    fi
-
-    opkg=$pkgver
-    if [ -z "$XBPS_KEEP_ALL" ]; then
-        remove_pkg_autodeps
-        remove_pkg_wrksrc
-        setup_pkg $sourcepkg $cross
-        remove_pkg $cross
-        remove_pkg_statedir
-    fi
-
-    # If base-chroot not installed, install "base-files" into masterdir
-    # from local repository; this is the only pkg required to be able to build
-    # the bootstrap pkgs from scratch.
-    if [ -z "$CHROOT_READY" -a "$pkgname" = "base-files" ]; then
-        msg_normal "Installing $opkg into masterdir...\n"
-        local _log=$(mktemp --tmpdir|| exit 1)
-        if [ -n "$XBPS_BUILD_FORCEMODE" ]; then
-            local _flags="-f"
-        fi
-        $XBPS_INSTALL_CMD ${_flags} -y $opkg >${_log} 2>&1
-        if [ $? -ne 0 ]; then
-            msg_red "Failed to install $opkg into masterdir, see below for errors:\n"
-            cat ${_log}
-            rm -f ${_log}
-            msg_error "Cannot continue!"
-        fi
-        rm -f ${_log}
-    fi
-
-    if [ "$XBPS_TARGET_PKG" = "$sourcepkg" ]; then
-        if [ "$XBPS_TARGET" = "bootstrap" ]; then
-            return 0
-        fi
-        # Package built successfully. Exit directly due to nested install_pkg
-        # and install_pkg_deps functions.
-        remove_cross_pkg $cross
-        exit_and_cleanup
-    fi
-}
-
 remove_pkg_autodeps() {
     local rval= tmplogf=
 

xbps-src

@@ -252,12 +252,16 @@ check_build_requirements() {
         if [ -n "${_gitbranch}" -a "${_gitbranch}" != "master" ]; then
             export XBPS_ALT_REPOSITORY="${_gitbranch}"
             export XBPS_REPOSITORY="${XBPS_REPOSITORY}/${_gitbranch}"
-            msg_normal "Using \`$XBPS_REPOSITORY\' as local repository.\n"
+            if [ -z "$IN_CHROOT" ]; then
+                msg_normal "Using \`$XBPS_REPOSITORY\' as local repository.\n"
+            fi
         fi
         popd &>/dev/null
     else
         export XBPS_REPOSITORY="${XBPS_REPOSITORY}/${XBPS_ALT_REPOSITORY}"
-        msg_normal "Using \`$XBPS_REPOSITORY\' as local repository.\n"
+        if [ -z "$IN_CHROOT" ]; then
+            msg_normal "Using \`$XBPS_REPOSITORY\' as local repository.\n"
+        fi
     fi
 }
 
@@ -462,8 +466,8 @@ export XBPS_FETCH_CMD="xbps-uhelper fetch"
 readonly XBPS_DIGEST_CMD="xbps-uhelper digest"
 readonly XBPS_CMPVER_CMD="xbps-uhelper cmpver"
 
-XBPS_TARGET="$1"
+readonly XBPS_TARGET="$1"
-XBPS_TARGET_PKG="${2##*/}"
+readonly XBPS_TARGET_PKG="${2##*/}"
 
 # Check if stdout is a tty; if false disable colors.
 test -t 1 || export NOCOLORS=1
@@ -500,7 +504,7 @@ export XBPS_SHUTILSDIR XBPS_CROSSPFDIR XBPS_TRIGGERSDIR \
     XBPS_SKIP_REMOTEREPOS XBPS_CROSS_BUILD XBPS_PKG_OPTIONS \
     XBPS_CONFIG_FILE XBPS_KEEP_ALL XBPS_HOSTDIR XBPS_MASTERDIR \
     XBPS_SRC_VERSION XBPS_DESTDIR XBPS_MACHINE XBPS_TEMP_MASTERDIR \
-    XBPS_BINPKG_EXISTS
+    XBPS_BINPKG_EXISTS XBPS_LIBEXECDIR
 
 for i in REPOSITORY DESTDIR BUILDDIR SRCDISTDIR; do
     eval val="\$XBPS_$i"
@@ -537,8 +541,9 @@ fi
 
 check_build_requirements
 
-trap 'exit_func' INT TERM
+if [ -z "$IN_CHROOT" ]; then
-
+    trap 'exit_func' INT TERM
+fi
 
 #
 # Main switch.
@@ -553,7 +558,6 @@ case "$XBPS_TARGET" in
         check_reqhost_utils bootstrap
         [ ! -d $XBPS_SRCPKGDIR/base-chroot ] && \
             msg_error "Cannot find $XBPS_SRCPKGDIR/base-chroot directory!\n"
-        XBPS_TARGET_PKG="base-chroot"
         bootstrap_vpkg=${XBPS_MASTERDIR}/etc/xbps.d/bootstrap-vpkgs.conf
         mkdir -p ${XBPS_MASTERDIR}/etc/xbps.d
         if [ ! -s ${bootstrap_vpkg} ]; then
@@ -561,11 +565,8 @@ case "$XBPS_TARGET" in
             echo 'virtualpkg=libgcc-4.4.0_1:base-files' >> ${bootstrap_vpkg}
             echo 'virtualpkg=libstdc++-4.4.0_1:base-files' >> ${bootstrap_vpkg}
         fi
-        BEGIN_INSTALL=1
+        $XBPS_LIBEXECDIR/build.sh base-chroot base-chroot $XBPS_TARGET || exit 1
-        setup_pkg $XBPS_TARGET_PKG
-        install_pkg $XBPS_TARGET
         [ -d $XBPS_MASTERDIR ] && rm -rf $XBPS_MASTERDIR
-        unset XBPS_TARGET_PKG
         install_bbootstrap
         ;;
     bootstrap-update)
@@ -605,7 +606,6 @@ case "$XBPS_TARGET" in
         fi
         ;;
     fetch|extract|build|configure|install|pkg)
-        BEGIN_INSTALL=1
         read_pkg
         if [ -n "$XBPS_TEMP_MASTERDIR" ]; then
             bootstrap_update
@@ -614,7 +614,8 @@ case "$XBPS_TARGET" in
         if [ -n "$CHROOT_READY" -a -z "$IN_CHROOT" ]; then
             chroot_handler $XBPS_TARGET $XBPS_TARGET_PKG
         else
-            install_pkg $XBPS_TARGET $XBPS_CROSS_BUILD
+            $XBPS_LIBEXECDIR/build.sh $XBPS_TARGET_PKG $XBPS_TARGET_PKG \
+                $XBPS_TARGET $XBPS_CROSS_BUILD || exit 1
         fi
         ;;
     remove|remove-destdir)