hooks/do-fetch/00-distfiles.sh: contents checksum

Add support for contents checksums. This is meant to be used for tarballs
like the ones coming from https://*.googlesource.com/ which on every download
change their outer checksum due to being packaged with different date/time
stamps or similar.

If such a distfile still contains the same files as the author of a template
found, then the checksum of its contents can be specified by prepending a
commercial at (@) to the corresponding checksum="..." line.

Depending on the format of the distfile its entire contents will then be
piped into sha256sum and the resulting digest is verified against what
follows after the @.

Manual.md

@@ -419,6 +419,13 @@ Example:
 separated by blanks. Please note that the order must be the same than
 was used in `${distfiles}`. Example `checksum="kkas00xjkjas"`
 
+If a distfile changes its checksum for every download because it is packaged
+on the fly on the server, like e.g. snapshot tarballs from any of the
+`https://*.googlesource.com/` sites, the checksum of the `archive contents`
+can be specified by prepending a commercial at (@).
+For tarballs you can find the contents checksum by using the command
+`tar xf <tarball.ext> --to-stdout | sha256sum`.
+
 - `wrksrc` The directory name where the package sources are extracted, by default
 set to `${pkgname}-${version}`.
 

common/hooks/do-fetch/00-distfiles.sh

@@ -20,23 +20,121 @@ get_cksum() {
 	echo "$cksum"
 }
 
+# Return the checksum of the contents of a tarball
+contents_cksum() {
+	local curfile="$1" cursufx cksum
+
+	case $curfile in
+	*.tar.lzma)   cursufx="txz";;
+	*.tar.lz)     cursufx="tlz";;
+	*.tlz)        cursufx="tlz";;
+	*.tar.xz)     cursufx="txz";;
+	*.txz)        cursufx="txz";;
+	*.tar.bz2)    cursufx="tbz";;
+	*.tbz)        cursufx="tbz";;
+	*.tar.gz)     cursufx="tgz";;
+	*.tgz)        cursufx="tgz";;
+	*.gz)         cursufx="gz";;
+	*.bz2)        cursufx="bz2";;
+	*.tar)        cursufx="tar";;
+	*.zip)        cursufx="zip";;
+	*.rpm)        cursufx="rpm";;
+	*.patch)      cursufx="txt";;
+	*.diff)       cursufx="txt";;
+	*.txt)        cursufx="txt";;
+	*.7z)	      cursufx="7z";;
+	*) msg_error "$pkgver: unknown distfile suffix for $curfile.\n";;
+	esac
+
+	case ${cursufx} in
+	tar|txz|tbz|tlz|tgz)
+		cksum=$(tar xf "$curfile" --to-stdout | sha256sum | awk '{print $1}')
+		if [ $? -ne 0 ]; then
+			msg_error "$pkgver: extracting $curfile to pipe.\n"
+		fi
+		;;
+	gz)
+		cksum=$(gunzip -c "$curfile" | sha256sum | awk '{print $1}')
+		;;
+	bz2)
+		cksum=$(bunzip2 -c "$curfile" | sha256sum | awk '{print $1}')
+		;;
+	zip)
+		if command -v unzip &>/dev/null; then
+			cksum=$(unzip -p "$curfile" | sha256sum | awk '{print $1}')
+			if [ $? -ne 0 ]; then
+				msg_error "$pkgver: extracting $curfile to pipe.\n"
+			fi
+		else
+			msg_error "$pkgver: cannot find unzip bin for extraction.\n"
+		fi
+		;;
+	rpm)
+		if command -v rpmextract &>/dev/null; then
+			cksum=$(rpm2cpio "$curfile" | bsdtar xf - --to-stdout | sha256sum | awk '{print $1}')
+			if [ $? -ne 0 ]; then
+				msg_error "$pkgver: extracting $curfile to pipe.\n"
+			fi
+		else
+			msg_error "$pkgver: cannot find rpmextract for extraction.\n"
+		fi
+		;;
+	txt)
+		cksum=$(cat "$curfile" | sha256sum | awk '{print $1}')
+		;;
+	7z)
+		if command -v 7z &>/dev/null; then
+			cksum=$(7z x -o "$curfile" | sha256sum | awk '{print $1}')
+			if [ $? -ne 0 ]; then
+				msg_error "$pkgver: extracting $curfile to pipe.\n"
+			fi
+		else
+			msg_error "$pkgver: cannot find 7z bin for extraction.\n"
+		fi
+		;;
+	*)
+		msg_error "$pkgver: cannot guess $curfile extract suffix. ($cursufx)\n"
+		;;
+	esac
+
+	if [ -z "$cksum" ]; then
+		msg_error "$pkgver: cannot find contents checksum for $curfile.\n"
+	fi
+	echo "$cksum"
+}
+
 # Verify the checksum for $curfile stored at $distfile and index $dfcount
 verify_cksum() {
 	local curfile="$1" distfile="$2" dfcount="$3" filesum cksum
 
 	cksum=$(get_cksum $curfile $dfcount)
-	msg_normal "$pkgver: verifying checksum for distfile '$curfile'... "
+
-	filesum=$(${XBPS_DIGEST_CMD} "$distfile")
+	# If the checksum starts with an commercial at (@) it is the contents checksum
-	if [ "$cksum" != "$filesum" ]; then
+	if [ "${cksum:0:1}" = "@" ]; then
-		echo
+		cksum=${cksum:1}
-		msg_red "SHA256 mismatch for '$curfile:'\n$filesum\n"
+		msg_normal "$pkgver: verifying contents checksum for distfile '$curfile'... "
-		errors=$((errors + 1))
+		filesum=$(contents_cksum "$curfile")
-	else
+		if [ "${cksum}" != "$filesum" ]; then
-		if [ ! -f "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}" ]; then
+			echo
-			mkdir -p "$XBPS_SRCDISTDIR/by_sha256"
+			msg_red "SHA256 mismatch for '$curfile:'\n@$filesum\n"
-			ln -f "$distfile" "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}"
+			errors=$((errors + 1))
+		else
+			msg_normal_append "OK.\n"
+		fi
+	else
+		msg_normal "$pkgver: verifying checksum for distfile '$curfile'... "
+		filesum=$(${XBPS_DIGEST_CMD} "$distfile")
+		if [ "$cksum" != "$filesum" ]; then
+			echo
+			msg_red "SHA256 mismatch for '$curfile:'\n$filesum\n"
+			errors=$((errors + 1))
+		else
+			if [ ! -f "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}" ]; then
+				mkdir -p "$XBPS_SRCDISTDIR/by_sha256"
+				ln -f "$distfile" "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}"
+			fi
+			msg_normal_append "OK.\n"
 		fi
-		msg_normal_append "OK.\n"
 	fi
 }
 
@@ -45,6 +143,7 @@ link_cksum() {
 	local curfile="$1" distfile="$2" dfcount="$3" filesum cksum
 
 	cksum=$(get_cksum $curfile $dfcount)
+
 	if [ -n "$cksum" -a -f "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}" ]; then
 		ln -f "$XBPS_SRCDISTDIR/by_sha256/${cksum}_${curfile}" "$distfile"
 		msg_normal "$pkgver: using known distfile $curfile.\n"
@@ -111,13 +210,18 @@ hook() {
 		curfile=$(basename "${f#*>}")
 		distfile="$srcdir/$curfile"
 		if [ -f "$distfile" ]; then
-			filesum=$(${XBPS_DIGEST_CMD} "$distfile")
 			cksum=$(get_cksum $curfile $dfcount)
+			if [ "${cksum:0:1}" = "@" ]; then
+				cksum=${cksum:1}
+				filesum=$(contents_cksum "$distfile")
+			else
+				filesum=$(${XBPS_DIGEST_CMD} "$distfile")
+			fi
 			if [ "$cksum" = "$filesum" ]; then
 				dfgood=$((dfgood + 1))
 			else
 				inode=$(stat "$distfile" --printf "%i")
-				msg_warn "$pkgver: wrong checksum found for ${curfile} - purging (inode: ${inode})\n"
+				msg_warn "$pkgver: wrong checksum found for ${curfile} - purging\n"
 				find ${XBPS_SRCDISTDIR} -inum ${inode} -delete -print
 			fi
 		fi