From 5dd4cef1a5a03dc180eb2ec1d2bfc97023f74d2b Mon Sep 17 00:00:00 2001
From: Matthias Klumpp <mak@debian.org>
Date: Sun, 23 Apr 2023 22:30:02 +0200
Subject: [PATCH] Implement support for repos with modern-style arch:all
 support

This implements support for modern APT repositories which have arch:any
and arch:all packages in separate Packages files, as outlined in the
Debian
repository documentation:
https://wiki.debian.org/DebianRepository/Format#Architectures
---
 functions | 274 +++++++++++++++++++++++++++++++++---------------------
 1 file changed, 168 insertions(+), 106 deletions(-)

diff --git a/functions b/functions
index 0ff5379..8bffbc6 100644
--- a/functions
+++ b/functions
@@ -557,6 +557,24 @@ extract_release_components () {
 	fi
 }
 
+repo_supports_arch_all () {
+	local a no_arch_all_support
+	local reldest="$1"; shift
+	TMPARCHS="$(sed -n 's/Architectures: *//p' "$reldest")"
+	ARCH_ALL_SUPPORTED=0
+	for a in $TMPARCHS ; do
+		if [ "$a" = "all" ]; then
+			ARCH_ALL_SUPPORTED=1
+			break
+		fi
+	done
+
+	no_arch_all_support=$(grep "^No-Support-for-Architecture-all: Packages$" "$reldest" || true)
+	if [ "$no_arch_all_support" != "" ]; then
+		ARCH_ALL_SUPPORTED=0
+	fi
+}
+
 CODENAME=""
 validate_suite () {
 	local reldest suite s
@@ -667,7 +685,7 @@ download_release_sig () {
 download_release_indices () {
 	local m1 inreldest reldest relsigdest totalpkgs \
 	      subpath xzi bz2i gzi normi i ext \
-	      donepkgs pkgdest acquirebyhash s c m
+	      donepkgs pkgdest acquirebyhash archs s c a m
 	m1="${MIRRORS%% *}"
 	for s in $SUITE $EXTRA_SUITES; do
 		inreldest="$TARGET/$($DLDEST rel "$s" "$m1" "dists/$s/InRelease")"
@@ -680,70 +698,80 @@ download_release_indices () {
 
 		extract_release_components "$reldest"
 
+		repo_supports_arch_all "$reldest"
+
+		archs="$ARCH"
+		if [ $ARCH_ALL_SUPPORTED -eq 1 ]; then
+			archs="all $ARCH"
+		fi
+
 		acquirebyhash=$(grep "^Acquire-By-Hash: yes$" "$reldest" || true)
-		totalpkgs=0
-		for c in $COMPONENTS; do
-			subpath="$c/binary-$ARCH/Packages"
-			xzi="$(get_release_checksum "$reldest" "$subpath.xz")"
-			bz2i="$(get_release_checksum "$reldest" "$subpath.bz2")"
-			gzi="$(get_release_checksum "$reldest" "$subpath.gz")"
-			normi="$(get_release_checksum "$reldest" "$subpath")"
-			if [ "$normi" != "" ]; then
-				i="$normi"
-			elif in_path bunzip2 && [ "$bz2i" != "" ]; then
-				i="$bz2i"
-			elif in_path unxz && [ "$xzi" != "" ]; then
-				i="$xzi"
-			elif in_path gunzip && [ "$gzi" != "" ]; then
-				i="$gzi"
-			fi
-			if [ "$i" != "" ]; then
-				totalpkgs=$(( $totalpkgs + ${i#* } ))
-			else
-				mv "$reldest" "$reldest.malformed"
-				error 1 MISSINGRELENTRY "Invalid Release file, no entry for %s" "$subpath"
-			fi
-		done
 
-		donepkgs=0
-		progress 0 $totalpkgs DOWNPKGS "Downloading Packages files"
-		for c in $COMPONENTS; do
-			subpath="$c/binary-$ARCH/Packages"
-			path="dists/$s/$subpath"
-			xzi="$(get_release_checksum "$reldest" "$subpath.xz")"
-			bz2i="$(get_release_checksum "$reldest" "$subpath.bz2")"
-			gzi="$(get_release_checksum "$reldest" "$subpath.gz")"
-			normi="$(get_release_checksum "$reldest" "$subpath")"
-			ext=""
-			if [ "$acquirebyhash" != "" ]; then
-				ext="$ext byhash"
-			fi
-			if [ "$normi" != "" ]; then
-				ext="$ext $normi ."
-				i="$normi"
-			fi
-			if in_path unxz && [ "$xzi" != "" ]; then
-				ext="$ext $xzi xz"
-				i="${i:-$xzi}"
-			fi
-			if in_path bunzip2 && [ "$bz2i" != "" ]; then
-				ext="$ext $bz2i bz2"
-				i="${i:-$bz2i}"
-			fi
-			if in_path gunzip && [ "$gzi" != "" ]; then
-				ext="$ext $gzi gz"
-				i="${i:-$gzi}"
-			fi
-			progress_next $(($donepkgs + ${i#* }))
-			for m in $MIRRORS; do
-				pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m" "$path")"
-				if get "$m/$path" "$pkgdest" $ext; then break; fi
+		for a in $archs; do
+			totalpkgs=0
+			for c in $COMPONENTS; do
+				subpath="$c/binary-$a/Packages"
+				xzi="$(get_release_checksum "$reldest" "$subpath.xz")"
+				bz2i="$(get_release_checksum "$reldest" "$subpath.bz2")"
+				gzi="$(get_release_checksum "$reldest" "$subpath.gz")"
+				normi="$(get_release_checksum "$reldest" "$subpath")"
+				if [ "$normi" != "" ]; then
+					i="$normi"
+				elif in_path bunzip2 && [ "$bz2i" != "" ]; then
+					i="$bz2i"
+				elif in_path unxz && [ "$xzi" != "" ]; then
+					i="$xzi"
+				elif in_path gunzip && [ "$gzi" != "" ]; then
+					i="$gzi"
+				fi
+				if [ "$i" != "" ]; then
+					totalpkgs=$(( $totalpkgs + ${i#* } ))
+				else
+					mv "$reldest" "$reldest.malformed"
+					error 1 MISSINGRELENTRY "Invalid Release file, no entry for %s" "$subpath"
+				fi
+			done
+
+			donepkgs=0
+			progress 0 $totalpkgs DOWNPKGS "Downloading Packages files for $a"
+			for c in $COMPONENTS; do
+				subpath="$c/binary-$a/Packages"
+				path="dists/$s/$subpath"
+				xzi="$(get_release_checksum "$reldest" "$subpath.xz")"
+				bz2i="$(get_release_checksum "$reldest" "$subpath.bz2")"
+				gzi="$(get_release_checksum "$reldest" "$subpath.gz")"
+				normi="$(get_release_checksum "$reldest" "$subpath")"
+				ext=""
+				if [ "$acquirebyhash" != "" ]; then
+					ext="$ext byhash"
+				fi
+				if [ "$normi" != "" ]; then
+					ext="$ext $normi ."
+					i="$normi"
+				fi
+				if in_path unxz && [ "$xzi" != "" ]; then
+					ext="$ext $xzi xz"
+					i="${i:-$xzi}"
+				fi
+				if in_path bunzip2 && [ "$bz2i" != "" ]; then
+					ext="$ext $bz2i bz2"
+					i="${i:-$bz2i}"
+				fi
+				if in_path gunzip && [ "$gzi" != "" ]; then
+					ext="$ext $gzi gz"
+					i="${i:-$gzi}"
+				fi
+				progress_next $(($donepkgs + ${i#* }))
+				for m in $MIRRORS; do
+					pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m" "$path")"
+					if get "$m/$path" "$pkgdest" $ext; then break; fi
+				done
+				if [ ! -f "$pkgdest" ]; then
+					error 1 COULDNTDL "Couldn't download %s" "$m/$path"
+				fi
+				donepkgs=$(($donepkgs + ${i#* }))
+				progress $donepkgs $totalpkgs DOWNPKGS "Downloading Packages files"
 			done
-			if [ ! -f "$pkgdest" ]; then
-				error 1 COULDNTDL "Couldn't download %s" "$m/$path"
-			fi
-			donepkgs=$(($donepkgs + ${i#* }))
-			progress $donepkgs $totalpkgs DOWNPKGS "Downloading Packages files"
 		done
 	done
 }
@@ -807,7 +835,7 @@ download_debs () {
 
 download_release () {
 	local m1 numdebs countdebs totaldebs leftoverdebs path pkgdest \
-	      dloaddebs s c m
+	      dloaddebs archs s c a m
 	m1="${MIRRORS%% *}"
 
 	numdebs="$#"
@@ -822,25 +850,32 @@ download_release () {
 	leftoverdebs=$(printf "$leftoverdebs"|tr ' ' '\n'|sort -u|tr '\n' ' ')
 	numdebs=$(printf "$leftoverdebs"|wc -w)
 
+	archs="$ARCH"
+	if [ $ARCH_ALL_SUPPORTED -eq 1 ]; then
+		archs="all $ARCH"
+	fi
+
 	for s in $SUITE $EXTRA_SUITES; do
-		for c in $COMPONENTS; do
-			if [ "$countdebs" -ge "$numdebs" ]; then break; fi
+		for a in $archs; do
+			for c in $COMPONENTS; do
+				if [ "$countdebs" -ge "$numdebs" ]; then break; fi
 
-			path="dists/$s/$c/binary-$ARCH/Packages"
-			pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")"
-			if [ ! -e "$pkgdest" ]; then continue; fi
+				path="dists/$s/$c/binary-$a/Packages"
+				pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m1" "$path")"
+				if [ ! -e "$pkgdest" ]; then continue; fi
 
-			info CHECKINGSIZES "Checking component %s on %s..." "$c" "$m1"
+				info CHECKINGSIZES "Checking component %s on %s..." "$c" "$m1"
 
-			leftoverdebs="$(get_package_sizes "$m1" "$pkgdest" $leftoverdebs)"
+				leftoverdebs="$(get_package_sizes "$m1" "$pkgdest" $leftoverdebs)"
 
-			countdebs=$(($countdebs + ${leftoverdebs%% *}))
-			leftoverdebs=${leftoverdebs#* }
+				countdebs=$(($countdebs + ${leftoverdebs%% *}))
+				leftoverdebs=${leftoverdebs#* }
 
-			totaldebs=${leftoverdebs%% *}
-			leftoverdebs=${leftoverdebs#* }
+				totaldebs=${leftoverdebs%% *}
+				leftoverdebs=${leftoverdebs#* }
 
-			progress "$countdebs" "$numdebs" SIZEDEBS "Finding package sizes"
+				progress "$countdebs" "$numdebs" SIZEDEBS "Finding package sizes"
+			done
 		done
 	done
 
@@ -855,15 +890,17 @@ download_release () {
 
 	pkgs_to_get="$*"
 	for s in $SUITE $EXTRA_SUITES; do
-		for c in $COMPONENTS; do
-			path="dists/$s/$c/binary-$ARCH/Packages"
-			for m in $MIRRORS; do
-				pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m" "$path")"
-				if [ ! -e "$pkgdest" ]; then continue; fi
-				pkgs_to_get="$(download_debs "$m" "$pkgdest" $pkgs_to_get 5>&1 1>&6)"
+		for a in $archs; do
+			for c in $COMPONENTS; do
+				path="dists/$s/$c/binary-$a/Packages"
+				for m in $MIRRORS; do
+					pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m" "$path")"
+					if [ ! -e "$pkgdest" ]; then continue; fi
+					pkgs_to_get="$(download_debs "$m" "$pkgdest" $pkgs_to_get 5>&1 1>&6)"
+					if [ -z "$pkgs_to_get" ]; then break; fi
+				done 6>&1
 				if [ -z "$pkgs_to_get" ]; then break; fi
-			done 6>&1
-			if [ -z "$pkgs_to_get" ]; then break; fi
+			done
 		done
 		if [ -z "$pkgs_to_get" ]; then break; fi
 	done
@@ -947,15 +984,23 @@ download_main () {
 ###################################################### deb choosing support
 
 get_debs () {
-	local field m1 s c path pkgdest
+	local field m1 s c a path pkgdest
 	field="$1"
 	shift
+
+	archs="$ARCH"
+	if [ $ARCH_ALL_SUPPORTED -eq 1 ]; then
+		archs="all $ARCH"
+	fi
+
 	for m1 in $MIRRORS; do
 		for s in $SUITE $EXTRA_SUITES; do
 			for c in $COMPONENTS; do
-				path="dists/$s/$c/binary-$ARCH/Packages"
-				pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")"
-				echo "$("$PKGDETAILS" FIELD "$field" "$m1" "$pkgdest" "$@" | sed 's/ .*//')"
+				for a in $archs; do
+					path="dists/$s/$c/binary-$a/Packages"
+					pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m1" "$path")"
+					echo "$("$PKGDETAILS" FIELD "$field" "$m1" "$pkgdest" "$@" | sed 's/ .*//')"
+				done
 			done
 		done
 	done
@@ -1587,14 +1632,22 @@ resolve_deps () {
 	local PKGS="$*"
 	local ALLPKGS="$PKGS";
 	local ALLPKGS2="";
-	local s c
+	local s c a
+
+	archs="$ARCH"
+	if [ $ARCH_ALL_SUPPORTED -eq 1 ]; then
+		archs="all $ARCH"
+	fi
+
 	while [ "$PKGS" != "" ]; do
 		local NEWPKGS=""
-		for s in $SUITE $EXTRA_SUITES; do
-			for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do
-				local path="dists/$s/$c/binary-$ARCH/Packages"
-				local pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")"
-				NEWPKGS="$NEWPKGS $("$PKGDETAILS" GETDEPS "$pkgdest" $PKGS)"
+		for a in $archs; do
+			for s in $SUITE $EXTRA_SUITES; do
+				for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do
+					local path="dists/$s/$c/binary-$a/Packages"
+					local pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m1" "$path")"
+					NEWPKGS="$NEWPKGS $("$PKGDETAILS" GETDEPS "$pkgdest" $PKGS)"
+				done
 			done
 		done
 		if [ -n "${EXCLUDE_DEPENDENCY:-}" ]; then
@@ -1602,11 +1655,13 @@ resolve_deps () {
 		fi
 		PKGS=$(echo "$PKGS $NEWPKGS" | tr ' ' '\n' | sort | uniq)
 		local REALPKGS=""
-		for s in $SUITE $EXTRA_SUITES; do
-			for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do
-				local path="dists/$s/$c/binary-$ARCH/Packages"
-				local pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")"
-				REALPKGS="$REALPKGS $("$PKGDETAILS" PKGS REAL "$pkgdest" $PKGS | sed -n 's/ .*REAL.*$//p')"
+		for a in $archs; do
+			for s in $SUITE $EXTRA_SUITES; do
+				for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do
+					local path="dists/$s/$c/binary-$a/Packages"
+					local pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m1" "$path")"
+					REALPKGS="$REALPKGS $("$PKGDETAILS" PKGS REAL "$pkgdest" $PKGS | sed -n 's/ .*REAL.*$//p')"
+				done
 			done
 		done
 		PKGS="$REALPKGS"
@@ -1621,15 +1676,22 @@ setup_available () {
 	local m1 c path pkgdest pkg
 	m1="${MIRRORS%% *}"
 
+	archs="$ARCH"
+	if [ $ARCH_ALL_SUPPORTED -eq 1 ]; then
+		archs="all $ARCH"
+	fi
+
 	for s in $SUITE $EXTRA_SUITES; do
-		for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do
-			path="dists/$s/$c/binary-$ARCH/Packages"
-			pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")"
-			# XXX: What if a package is in more than one component?
-			# -- cjwatson 2009-07-29
-			# XXX: ...or suite?
-			# -- jrtc27 2019-06-11
-			"$PKGDETAILS" STANZAS "$pkgdest" "$@"
+		for a in $archs; do
+			for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do
+				path="dists/$s/$c/binary-$a/Packages"
+				pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m1" "$path")"
+				# XXX: What if a package is in more than one component?
+				# -- cjwatson 2009-07-29
+				# XXX: ...or suite?
+				# -- jrtc27 2019-06-11
+				"$PKGDETAILS" STANZAS "$pkgdest" "$@"
+			done
 		done
 	done >"$TARGET/var/lib/dpkg/available"
 
-- 
GitLab

