From f4544b82a6d53a413bd093ec0bb490e21c04801a Mon Sep 17 00:00:00 2001 From: Enno Boland Date: Mon, 11 Aug 2014 18:05:13 +0200 Subject: [PATCH 1/4] common/hooks: add support for vcdiff creation --- common/hooks/post-pkg/01-xdelta_repolist.sh | 24 +++++++++++++++++++++ common/hooks/pre-pkg/01-xdelta_repolist.sh | 11 ++++++++++ xbps-src | 3 ++- 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 common/hooks/post-pkg/01-xdelta_repolist.sh create mode 100644 common/hooks/pre-pkg/01-xdelta_repolist.sh diff --git a/common/hooks/post-pkg/01-xdelta_repolist.sh b/common/hooks/post-pkg/01-xdelta_repolist.sh new file mode 100644 index 00000000000..a18f6ae3071 --- /dev/null +++ b/common/hooks/post-pkg/01-xdelta_repolist.sh @@ -0,0 +1,24 @@ +# This hook generates vcdiffs + +hook() { + set -x + [ -z "$XBPS_GENERATE_VCDIFF" ] && return 0; + + find $XBPS_REPOSITORY -name '*.genVcdiff' | xargs -r sha256sum | \ + while read chk oldfile; do + newfile=${oldfile/.genVcdiff/} + + if ! cmp -s "${newfile}" "${oldfile}"; then + newdiff="${newfile}.${chk}.vcdiff" + xdelta3 -D -R -f -e -s "${oldfile}" "${newfile}" "${newdiff}" + for diff in ${newfile}.*.vcdiff; do + [ "${diff}" = "${newdiff}" ] && continue; + cp -- "${diff}" "${diff}.tmp" + xdelta3 -f merge -m "${diff}.tmp" "${newdiff}" "${diff}" + rm -- "${diff}.tmp" + done + fi + + rm -- "${oldfile}" + done +} diff --git a/common/hooks/pre-pkg/01-xdelta_repolist.sh b/common/hooks/pre-pkg/01-xdelta_repolist.sh new file mode 100644 index 00000000000..6d6b21d739a --- /dev/null +++ b/common/hooks/pre-pkg/01-xdelta_repolist.sh @@ -0,0 +1,11 @@ +# this hook marks files which are about to change for generating vcdiffs + +hook() { + [ -z "$XBPS_GENERATE_VCDIFF" ] && return 0; + + # create links to preserve old versions of repodata + find $XBPS_REPOSITORY -name '*-repodata' | \ + while read; do + ln "${REPLY}" "${REPLY}.genVcdiff" + done +} diff --git a/xbps-src b/xbps-src index dc3535ee677..684e19aa149 100755 --- a/xbps-src +++ b/xbps-src @@ -469,7 +469,8 @@ export XBPS_SHUTILSDIR XBPS_CROSSPFDIR XBPS_TRIGGERSDIR \ XBPS_CCACHE XBPS_DISTCC XBPS_DISTCC_HOSTS XBPS_SKIP_DEPS \ XBPS_SKIP_REMOTEREPOS XBPS_CROSS_BUILD XBPS_PKG_OPTIONS \ XBPS_CONFIG_FILE XBPS_KEEP_ALL XBPS_HOSTDIR XBPS_MASTERDIR \ - XBPS_SRC_VERSION XBPS_DESTDIR FAKEROOT_CMD CHROOT_CMD XBPS_MACHINE + XBPS_SRC_VERSION XBPS_DESTDIR FAKEROOT_CMD CHROOT_CMD XBPS_MACHINE \ + XBPS_GENERATE_VCDIFF for i in REPOSITORY DESTDIR BUILDDIR SRCDISTDIR; do eval val="\$XBPS_$i" From b713c6241359f718a4a84ca1b36210f84d8293eb Mon Sep 17 00:00:00 2001 From: Enno Boland Date: Mon, 11 Aug 2014 19:55:14 +0200 Subject: [PATCH 2/4] common/hooks: generate null diffs to the new files itself. --- common/hooks/post-pkg/01-xdelta_repolist.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/common/hooks/post-pkg/01-xdelta_repolist.sh b/common/hooks/post-pkg/01-xdelta_repolist.sh index a18f6ae3071..d5f2099b3f6 100644 --- a/common/hooks/post-pkg/01-xdelta_repolist.sh +++ b/common/hooks/post-pkg/01-xdelta_repolist.sh @@ -19,6 +19,11 @@ hook() { done fi + # generate an empty diff to the new file + newchk=`sha256sum ${newfile} | awk '{ print $1 }'` + xdelta3 -D -R -f -e -s "${newfile}" "${newfile}" \ + "${newfile}.${newchk}.vcdiff" + rm -- "${oldfile}" done } From 27e96f018ba9385683f36f570e1f7bf495e6c5c7 Mon Sep 17 00:00:00 2001 From: Enno Boland Date: Tue, 12 Aug 2014 08:28:43 +0200 Subject: [PATCH 3/4] common/hooks: enable repackaging of indexes for xdelta --- common/hooks/post-pkg/01-xdelta_repolist.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/hooks/post-pkg/01-xdelta_repolist.sh b/common/hooks/post-pkg/01-xdelta_repolist.sh index d5f2099b3f6..b2b9846ce08 100644 --- a/common/hooks/post-pkg/01-xdelta_repolist.sh +++ b/common/hooks/post-pkg/01-xdelta_repolist.sh @@ -10,7 +10,7 @@ hook() { if ! cmp -s "${newfile}" "${oldfile}"; then newdiff="${newfile}.${chk}.vcdiff" - xdelta3 -D -R -f -e -s "${oldfile}" "${newfile}" "${newdiff}" + xdelta3 -f -e -s "${oldfile}" "${newfile}" "${newdiff}" for diff in ${newfile}.*.vcdiff; do [ "${diff}" = "${newdiff}" ] && continue; cp -- "${diff}" "${diff}.tmp" @@ -21,7 +21,7 @@ hook() { # generate an empty diff to the new file newchk=`sha256sum ${newfile} | awk '{ print $1 }'` - xdelta3 -D -R -f -e -s "${newfile}" "${newfile}" \ + xdelta3 -f -e -s "${newfile}" "${newfile}" \ "${newfile}.${newchk}.vcdiff" rm -- "${oldfile}" From bdb339f5fa0044ac9b9a1fd296b581b41be5ff9e Mon Sep 17 00:00:00 2001 From: Enno Boland Date: Tue, 12 Aug 2014 09:42:05 +0200 Subject: [PATCH 4/4] common/hooks: remove config variable enable vcdiff generation. --- common/hooks/post-pkg/01-xdelta_repolist.sh | 2 +- common/hooks/pre-pkg/01-xdelta_repolist.sh | 2 +- xbps-src | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/common/hooks/post-pkg/01-xdelta_repolist.sh b/common/hooks/post-pkg/01-xdelta_repolist.sh index b2b9846ce08..24c961a470f 100644 --- a/common/hooks/post-pkg/01-xdelta_repolist.sh +++ b/common/hooks/post-pkg/01-xdelta_repolist.sh @@ -2,7 +2,7 @@ hook() { set -x - [ -z "$XBPS_GENERATE_VCDIFF" ] && return 0; + type -P xdelta3 > /dev/null || return 0; find $XBPS_REPOSITORY -name '*.genVcdiff' | xargs -r sha256sum | \ while read chk oldfile; do diff --git a/common/hooks/pre-pkg/01-xdelta_repolist.sh b/common/hooks/pre-pkg/01-xdelta_repolist.sh index 6d6b21d739a..4a3941885e2 100644 --- a/common/hooks/pre-pkg/01-xdelta_repolist.sh +++ b/common/hooks/pre-pkg/01-xdelta_repolist.sh @@ -1,7 +1,7 @@ # this hook marks files which are about to change for generating vcdiffs hook() { - [ -z "$XBPS_GENERATE_VCDIFF" ] && return 0; + type -P xdelta3 > /dev/null || return 0; # create links to preserve old versions of repodata find $XBPS_REPOSITORY -name '*-repodata' | \ diff --git a/xbps-src b/xbps-src index 684e19aa149..dc3535ee677 100755 --- a/xbps-src +++ b/xbps-src @@ -469,8 +469,7 @@ export XBPS_SHUTILSDIR XBPS_CROSSPFDIR XBPS_TRIGGERSDIR \ XBPS_CCACHE XBPS_DISTCC XBPS_DISTCC_HOSTS XBPS_SKIP_DEPS \ XBPS_SKIP_REMOTEREPOS XBPS_CROSS_BUILD XBPS_PKG_OPTIONS \ XBPS_CONFIG_FILE XBPS_KEEP_ALL XBPS_HOSTDIR XBPS_MASTERDIR \ - XBPS_SRC_VERSION XBPS_DESTDIR FAKEROOT_CMD CHROOT_CMD XBPS_MACHINE \ - XBPS_GENERATE_VCDIFF + XBPS_SRC_VERSION XBPS_DESTDIR FAKEROOT_CMD CHROOT_CMD XBPS_MACHINE for i in REPOSITORY DESTDIR BUILDDIR SRCDISTDIR; do eval val="\$XBPS_$i"