From c1fd6c389ef8678e223a9ca1801be32633794a55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Buchm=C3=BCller?= Date: Fri, 2 Mar 2018 23:07:30 +0100 Subject: [PATCH] hooks/do-fetch/00-distfiles.sh: contents checksum Add support for contents checksums. This is meant to be used for tarballs like the ones coming from https://*.googlesource.com/ which on every download change their outer checksum due to being packaged with different date/time stamps or similar. If such a distfile still contains the same files as the author of a template found, then the checksum of its contents can be specified by prepending a commercial at (@) to the corresponding checksum="..." line. Depending on the format of the distfile its entire contents will then be piped into sha256sum and the resulting digest is verified against what follows after the @. --- Manual.md | 7 ++ common/hooks/do-fetch/00-distfiles.sh | 130 +++++++++++++++++++++++--- 2 files changed, 124 insertions(+), 13 deletions(-) diff --git a/Manual.md b/Manual.md index 1c68481a8b5..04a4315ce39 100644 --- a/Manual.md +++ b/Manual.md @@ -419,6 +419,13 @@ Example: separated by blanks. Please note that the order must be the same than was used in `${distfiles}`. Example `checksum="kkas00xjkjas"` +If a distfile changes its checksum for every download because it is packaged +on the fly on the server, like e.g. snapshot tarballs from any of the +`https://*.googlesource.com/` sites, the checksum of the `archive contents` +can be specified by prepending a commercial at (@). +For tarballs you can find the contents checksum by using the command +`tar xf --to-stdout | sha256sum`. + - `wrksrc` The directory name where the package sources are extracted, by default set to `${pkgname}-${version}`. diff --git a/common/hooks/do-fetch/00-distfiles.sh b/common/hooks/do-fetch/00-distfiles.sh index 5e0006789b4..09bac46fa69 100644 --- a/common/hooks/do-fetch/00-distfiles.sh +++ b/common/hooks/do-fetch/00-distfiles.sh @@ -20,23 +20,121 @@ get_cksum() { echo "$cksum" } +# Return the checksum of the contents of a tarball +contents_cksum() { + local curfile="$1" cursufx cksum + + case $curfile in + *.tar.lzma) cursufx="txz";; + *.tar.lz) cursufx="tlz";; + *.tlz) cursufx="tlz";; + *.tar.xz) cursufx="txz";; + *.txz) cursufx="txz";; + *.tar.bz2) cursufx="tbz";; + *.tbz) cursufx="tbz";; + *.tar.gz) cursufx="tgz";; + *.tgz) cursufx="tgz";; + *.gz) cursufx="gz";; + *.bz2) cursufx="bz2";; + *.tar) cursufx="tar";; + *.zip) cursufx="zip";; + *.rpm) cursufx="rpm";; + *.patch) cursufx="txt";; + *.diff) cursufx="txt";; + *.txt) cursufx="txt";; + *.7z) cursufx="7z";; + *) msg_error "$pkgver: unknown distfile suffix for $curfile.\n";; + esac + + case ${cursufx} in + tar|txz|tbz|tlz|tgz) + cksum=$(tar xf "$curfile" --to-stdout | sha256sum | awk '{print $1}') + if [ $? -ne 0 ]; then + msg_error "$pkgver: extracting $curfile to pipe.\n" + fi + ;; + gz) + cksum=$(gunzip -c "$curfile" | sha256sum | awk '{print $1}') + ;; + bz2) + cksum=$(bunzip2 -c "$curfile" | sha256sum | awk '{print $1}') + ;; + zip) + if command -v unzip &>/dev/null; then + cksum=$(unzip -p "$curfile" | sha256sum | awk '{print $1}') + if [ $? -ne 0 ]; then + msg_error "$pkgver: extracting $curfile to pipe.\n" + fi + else + msg_error "$pkgver: cannot find unzip bin for extraction.\n" + fi + ;; + rpm) + if command -v rpmextract &>/dev/null; then + cksum=$(rpm2cpio "$curfile" | bsdtar xf - --to-stdout | sha256sum | awk '{print $1}') + if [ $? -ne 0 ]; then + msg_error "$pkgver: extracting $curfile to pipe.\n" + fi + else + msg_error "$pkgver: cannot find rpmextract for extraction.\n" + fi + ;; + txt) + cksum=$(cat "$curfile" | sha256sum | awk '{print $1}') + ;; + 7z) + if command -v 7z &>/dev/null; then + cksum=$(7z x -o "$curfile" | sha256sum | awk '{print $1}') + if [ $? -ne 0 ]; then + msg_error "$pkgver: extracting $curfile to pipe.\n" + fi + else + msg_error "$pkgver: cannot find 7z bin for extraction.\n" + fi + ;; + *) + msg_error "$pkgver: cannot guess $curfile extract suffix. ($cursufx)\n" + ;; + esac + + if [ -z "$cksum" ]; then + msg_error "$pkgver: cannot find contents checksum for $curfile.\n" + fi + echo "$cksum" +} + # Verify the checksum for $curfile stored at $distfile and index $dfcount verify_cksum() { local curfile="$1" distfile="$2" dfcount="$3" filesum cksum cksum=$(get_cksum $curfile $dfcount) - msg_normal "$pkgver: verifying checksum for distfile '$curfile'... " - filesum=$(${XBPS_DIGEST_CMD} "$distfile") - if [ "$cksum" != "$filesum" ]; then - echo - msg_red "SHA256 mismatch for '$curfile:'\n$filesum\n" - errors=$((errors + 1)) - else - if [ ! -f "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}" ]; then - mkdir -p "$XBPS_SRCDISTDIR/by_sha256" - ln -f "$distfile" "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}" + + # If the checksum starts with an commercial at (@) it is the contents checksum + if [ "${cksum:0:1}" = "@" ]; then + cksum=${cksum:1} + msg_normal "$pkgver: verifying contents checksum for distfile '$curfile'... " + filesum=$(contents_cksum "$curfile") + if [ "${cksum}" != "$filesum" ]; then + echo + msg_red "SHA256 mismatch for '$curfile:'\n@$filesum\n" + errors=$((errors + 1)) + else + msg_normal_append "OK.\n" + fi + else + msg_normal "$pkgver: verifying checksum for distfile '$curfile'... " + filesum=$(${XBPS_DIGEST_CMD} "$distfile") + if [ "$cksum" != "$filesum" ]; then + echo + msg_red "SHA256 mismatch for '$curfile:'\n$filesum\n" + errors=$((errors + 1)) + else + if [ ! -f "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}" ]; then + mkdir -p "$XBPS_SRCDISTDIR/by_sha256" + ln -f "$distfile" "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}" + fi + msg_normal_append "OK.\n" fi - msg_normal_append "OK.\n" fi } @@ -45,6 +143,7 @@ link_cksum() { local curfile="$1" distfile="$2" dfcount="$3" filesum cksum cksum=$(get_cksum $curfile $dfcount) + if [ -n "$cksum" -a -f "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}" ]; then ln -f "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}" "$distfile" msg_normal "$pkgver: using known distfile $curfile.\n" @@ -111,13 +210,18 @@ hook() { curfile=$(basename "${f#*>}") distfile="$srcdir/$curfile" if [ -f "$distfile" ]; then - filesum=$(${XBPS_DIGEST_CMD} "$distfile") cksum=$(get_cksum $curfile $dfcount) + if [ "${cksum:0:1}" = "@" ]; then + cksum=${cksum:1} + filesum=$(contents_cksum "$distfile") + else + filesum=$(${XBPS_DIGEST_CMD} "$distfile") + fi if [ "$cksum" = "$filesum" ]; then dfgood=$((dfgood + 1)) else inode=$(stat "$distfile" --printf "%i") - msg_warn "$pkgver: wrong checksum found for ${curfile} - purging (inode: ${inode})\n" + msg_warn "$pkgver: wrong checksum found for ${curfile} - purging\n" find ${XBPS_SRCDISTDIR} -inum ${inode} -delete -print fi fi