Add build2/download scriptHEADmaster

1 files changed, 334 insertions, 0 deletions

diff --git a/build2/download b/build2/download
new file mode 100755
index 0000000..8851c12
--- /dev/null
+++ b/build2/download
@@ -0,0 +1,334 @@
+#! /usr/bin/env bash
+
+# Download and arrange source and binary packages.
+#
+# Run from the root of the destination download directory. Note that the
+# destination files are expected not to exist.
+
+repo="https://queue.stage.build2.org"
+#mingw="https://download.build2.org/0.17.0/build2-mingw-0.17.0-x86_64-windows.tar.xz"
+mingw="https://stage.build2.org/0/0.18.0-a.0/build2-mingw-0.18.0-a.0-x86_64-windows.tar.xz"
+
+bin_repo="$repo/0/bindist"
+src_repo="$repo/1"
+
+declare -A vers
+declare -A srcs
+declare -A bins
+
+# Project to version map.
+#
+# Key is project name, value is space-separated list of the
+# '[<src-repo-url>/]<repo-section>/<version>' entries. For entries with the
+# omitted '<src-repo-url>/' prefix the "$src_repo/" prefix is assumed. For
+# example:
+#
+# alpha/3.3.1-a.0.20241206125527.c3e2855e0a52 https://pkg.cppget.org/1/stable/1.1.1+21
+#
+# Note that for the binary distribution packages only the first version is
+# used.
+#
+# Note that currently we download two versions of openssl libraries since
+# libmysqlclient 8.0.15+18 constrains the libcrypto and libssl dependency
+# versions with ^1.1.1.
+#
+vers["odb"]="testing/2.5.0"
+vers["mysql"]="https://pkg.cppget.org/1/stable/8.0.15+18"
+vers["sqlite"]="https://pkg.cppget.org/1/stable/3.45.3+1"
+vers["postgresql"]="https://pkg.cppget.org/1/stable/14.1.0+6"
+vers["libcutl"]="https://pkg.cppget.org/1/stable/1.11.0"
+vers["libstudxml"]="https://pkg.cppget.org/1/testing/1.1.0"
+vers["zlib"]="https://pkg.cppget.org/1/stable/1.2.1200+5"
+vers["zstd"]="https://pkg.cppget.org/1/stable/1.5.5+1"
+vers["openssl"]="https://pkg.cppget.org/1/stable/3.3.1 https://pkg.cppget.org/1/stable/1.1.1+21"
+vers["isocline"]="https://pkg.cppget.org/1/stable/1.0.9+1"
+
+# Package to project map.
+#
+# Key is package name, value is project name (for all specified versions).
+#
+# Note that we don't download the Boost and Qt source packages, which may
+# potentially be required for building odb-tests and odb-examples, since there
+# are too large.
+#
+srcs["libodb"]="odb"
+srcs["libodb-mysql"]="odb"
+srcs["libodb-sqlite"]="odb"
+srcs["libodb-pgsql"]="odb"
+srcs["libodb-oracle"]="odb"
+srcs["libodb-mssql"]="odb"
+srcs["libodb-boost"]="odb"
+srcs["libodb-qt"]="odb"
+srcs["odb"]="odb"
+srcs["odb-tests"]="odb"
+srcs["odb-examples"]="odb"
+srcs["libmysqlclient"]="mysql"
+srcs["mysql-client"]="mysql"
+srcs["libsqlite3"]="sqlite"
+srcs["sqlite3"]="sqlite"
+srcs["libpq"]="postgresql"
+srcs["psql"]="postgresql"
+srcs["libcutl"]="libcutl"
+srcs["libstudxml"]="libstudxml"
+srcs["libz"]="zlib"
+srcs["libzstd"]="zstd"
+srcs["libcrypto"]="openssl"
+srcs["libssl"]="openssl"
+srcs["libisocline"]="isocline"
+
+# Binary source to destination path map.
+#
+# The key is the "<dist>/<os>/<proj>/<pkg>/<cfg>" bindist artifact path. The
+# value is the destination subdirectory.
+#
+# Note that <cfg> is a sanitized package configuration name prefixed with an
+# architecture (see brep's upload-bindist handler for details).
+#
+bins["debian/debian11/odb/libodb/x86_64"]="debian/debian11/x86_64"
+bins["debian/debian11/odb/libodb-mysql/x86_64"]="debian/debian11/x86_64"
+bins["debian/debian11/odb/libodb-sqlite/x86_64"]="debian/debian11/x86_64"
+bins["debian/debian11/odb/libodb-pgsql/x86_64"]="debian/debian11/x86_64"
+bins["debian/debian11/odb/libodb-boost/x86_64"]="debian/debian11/x86_64"
+bins["debian/debian11/odb/libodb-qt/x86_64"]="debian/debian11/x86_64"
+bins["debian/debian11/odb/odb/x86_64"]="debian/debian11/x86_64"
+
+bins["debian/debian12/odb/libodb/x86_64"]="debian/debian12/x86_64"
+bins["debian/debian12/odb/libodb-mysql/x86_64"]="debian/debian12/x86_64"
+bins["debian/debian12/odb/libodb-sqlite/x86_64"]="debian/debian12/x86_64"
+bins["debian/debian12/odb/libodb-pgsql/x86_64"]="debian/debian12/x86_64"
+bins["debian/debian12/odb/libodb-boost/x86_64"]="debian/debian12/x86_64"
+bins["debian/debian12/odb/libodb-qt/x86_64"]="debian/debian12/x86_64"
+bins["debian/debian12/odb/odb/x86_64"]="debian/debian12/x86_64"
+
+bins["debian/ubuntu22.04/odb/libodb/x86_64"]="ubuntu/ubuntu22.04/x86_64"
+bins["debian/ubuntu22.04/odb/libodb-mysql/x86_64"]="ubuntu/ubuntu22.04/x86_64"
+bins["debian/ubuntu22.04/odb/libodb-sqlite/x86_64"]="ubuntu/ubuntu22.04/x86_64"
+bins["debian/ubuntu22.04/odb/libodb-pgsql/x86_64"]="ubuntu/ubuntu22.04/x86_64"
+bins["debian/ubuntu22.04/odb/libodb-boost/x86_64"]="ubuntu/ubuntu22.04/x86_64"
+bins["debian/ubuntu22.04/odb/libodb-qt/x86_64"]="ubuntu/ubuntu22.04/x86_64"
+bins["debian/ubuntu22.04/odb/odb/x86_64"]="ubuntu/ubuntu22.04/x86_64"
+
+bins["debian/ubuntu24.04/odb/libodb/x86_64"]="ubuntu/ubuntu24.04/x86_64"
+bins["debian/ubuntu24.04/odb/libodb-mysql/x86_64"]="ubuntu/ubuntu24.04/x86_64"
+bins["debian/ubuntu24.04/odb/libodb-sqlite/x86_64"]="ubuntu/ubuntu24.04/x86_64"
+bins["debian/ubuntu24.04/odb/libodb-pgsql/x86_64"]="ubuntu/ubuntu24.04/x86_64"
+bins["debian/ubuntu24.04/odb/libodb-boost/x86_64"]="ubuntu/ubuntu24.04/x86_64"
+bins["debian/ubuntu24.04/odb/libodb-qt/x86_64"]="ubuntu/ubuntu24.04/x86_64"
+bins["debian/ubuntu24.04/odb/odb/x86_64"]="ubuntu/ubuntu24.04/x86_64"
+
+bins["fedora/fedora40/odb/libodb/x86_64"]="fedora/fedora40/x86_64"
+bins["fedora/fedora40/odb/libodb-mysql/x86_64"]="fedora/fedora40/x86_64"
+bins["fedora/fedora40/odb/libodb-sqlite/x86_64"]="fedora/fedora40/x86_64"
+bins["fedora/fedora40/odb/libodb-pgsql/x86_64"]="fedora/fedora40/x86_64"
+bins["fedora/fedora40/odb/libodb-boost/x86_64"]="fedora/fedora40/x86_64"
+bins["fedora/fedora40/odb/libodb-qt/x86_64"]="fedora/fedora40/x86_64"
+bins["fedora/fedora40/odb/odb/x86_64"]="fedora/fedora40/x86_64"
+
+bins["fedora/fedora41/odb/libodb/x86_64"]="fedora/fedora41/x86_64"
+bins["fedora/fedora41/odb/libodb-mysql/x86_64"]="fedora/fedora41/x86_64"
+bins["fedora/fedora41/odb/libodb-sqlite/x86_64"]="fedora/fedora41/x86_64"
+bins["fedora/fedora41/odb/libodb-pgsql/x86_64"]="fedora/fedora41/x86_64"
+bins["fedora/fedora41/odb/libodb-boost/x86_64"]="fedora/fedora41/x86_64"
+bins["fedora/fedora41/odb/libodb-qt/x86_64"]="fedora/fedora41/x86_64"
+bins["fedora/fedora41/odb/odb/x86_64"]="fedora/fedora41/x86_64"
+
+bins["fedora/rhel9.2/odb/libodb/x86_64"]="rhel/rhel9.2/x86_64"
+bins["fedora/rhel9.2/odb/libodb-mysql/x86_64"]="rhel/rhel9.2/x86_64"
+bins["fedora/rhel9.2/odb/libodb-sqlite/x86_64"]="rhel/rhel9.2/x86_64"
+bins["fedora/rhel9.2/odb/libodb-pgsql/x86_64"]="rhel/rhel9.2/x86_64"
+bins["fedora/rhel9.2/odb/libodb-boost/x86_64"]="rhel/rhel9.2/x86_64"
+bins["fedora/rhel9.2/odb/libodb-qt/x86_64"]="rhel/rhel9.2/x86_64"
+bins["fedora/rhel9.2/odb/odb/x86_64"]="rhel/rhel9.2/x86_64"
+
+bins["archive/windows10/odb/libodb/x86_64-release"]="windows/windows10/x86_64"
+bins["archive/windows10/odb/libodb/x86_64-debug"]="windows/windows10/x86_64"
+bins["archive/windows10/odb/libodb-mysql/x86_64-release"]="windows/windows10/x86_64"
+bins["archive/windows10/odb/libodb-mysql/x86_64-debug"]="windows/windows10/x86_64"
+bins["archive/windows10/odb/libodb-sqlite/x86_64-release"]="windows/windows10/x86_64"
+bins["archive/windows10/odb/libodb-sqlite/x86_64-debug"]="windows/windows10/x86_64"
+bins["archive/windows10/odb/libodb-pgsql/x86_64-release"]="windows/windows10/x86_64"
+bins["archive/windows10/odb/libodb-pgsql/x86_64-debug"]="windows/windows10/x86_64"
+bins["archive/windows10/odb/libodb-boost/x86_64-release"]="windows/windows10/x86_64"
+bins["archive/windows10/odb/libodb-boost/x86_64-debug"]="windows/windows10/x86_64"
+bins["archive/windows10/odb/libodb-qt/x86_64-release"]="windows/windows10/x86_64"
+bins["archive/windows10/odb/libodb-qt/x86_64-debug"]="windows/windows10/x86_64"
+bins["archive/windows10/odb/odb/x86_64"]="windows/windows10/x86_64"
+
+owd=`pwd`
+trap "{ cd $owd; exit 1; }" ERR
+set -o errtrace   # Trap in functions.
+set -o pipefail   # Fail if any pipeline command fails.
+shopt -s lastpipe # Execute last pipeline command in current shell.
+
+function info () { echo "$*" 1>&2; }
+function error () { info "$*"; exit 1; }
+
+declare -A dirs # Package directories to generate packages.sha256 in.
+
+# Download and arrange sources.
+#
+if true; then
+for pkg in "${!srcs[@]}"; do
+  prj="${srcs[$pkg]}"
+
+  for v in ${vers["$prj"]}; do
+
+    ver="$(sed -rn -e 's#^.+/([^/]+)$#\1#p' <<<"$v")"
+    sec="$(sed -rn -e 's#^(.+/)?([^/]+)/[^/]+$#\2#p' <<<"$v")"
+    rep="$(sed -rn -e 's#^((.+)/)?[^/]+/[^/]+$#\2#p' <<<"$v")"
+
+    if [ -z "$rep" ]; then
+      rep="$src_repo"
+    fi
+
+    pkgf="$pkg-$ver.tar.gz"
+    url="$rep/$sec/$prj/$pkgf"
+
+    info "fetching $url"
+    curl -sSf -o "$pkgf" "$url"
+
+    # Make .zip archive for examples.
+    #
+    if [ -n "$(sed -rn -e 's/^.+-examples$/true/p' <<<"$pkg")" ]; then
+      pkgd="$pkg-$ver"
+
+      if [ -d "$pkgd" ]; then
+        error "package directory $pkgd already exist"
+      fi
+
+      info "repacking $pkgf"
+      tar -xf "$pkgf"
+      zip -9 -rq "$pkg-$ver.zip" "$pkgd"
+      rm -rf "$pkgd"
+    fi
+  done
+done
+dirs["."]=true
+fi
+
+# Download and arrange binaries.
+#
+# Also overlay MinGW distribution over the ODB compiler archive for Windows.
+#
+if true; then
+for bk in "${!bins[@]}"; do
+  bv="${bins[$bk]}"
+
+  # Get the distribution type we are dealing with.
+  #
+  dist="$(sed -rn -e 's#^(.+)/.+/.+/.+/.+$#\1#p' <<<"$bk")"
+
+  # Get the project we are dealing with and its version. Pick the first
+  # version from potentially multiple ones specified for the project.
+  #
+  prj="$(sed -rn -e 's#^.+/.+/(.+)/.+/.+$#\1#p' <<<"$bk")"
+
+  v="$(sed -rn -e 's#^([^ ]+).*$#\1#p' <<<"${vers[$prj]}")"
+  ver="$(sed -rn -e 's#^.+/([^/]+)$#\1#p' <<<"$v")"
+
+  url="$bin_repo/"
+  url+="$(sed -rn -e "s#^(.+/.+/.+/.+)/(.+)\$#\\1/$ver/\\2#p" <<<"$bk")"
+
+  # Get the list of packages (keys) and their checksums (values) at this URL
+  # by loading its packages.sha256.
+  #
+  unset pkgs # Clear.
+  declare -A pkgs
+
+  info "fetching $url/packages.sha256"
+  curl -sSf "$url/packages.sha256" | while read l; do
+
+    pkg="$(sed -rn -e 's#^([^ ]+) \*(.+)$#\2#p' <<<"$l")"
+
+    # Get the extension and omit certain distribution-specific file types.
+    #
+    e="$(sed -rn -e 's#^.+\.([^.]+)$#\1#p' <<<"$pkg")"
+
+    case "$dist" in
+      "debian")
+        if [ "$e" = "buildinfo" -o "$e" = "changes" ]; then
+          continue
+        fi
+	      ;;
+
+      "fedora")
+        ;;
+    esac
+
+    csm="$(sed -rn -e 's#^([^ ]+) \*(.+)$#\1#p' <<<"$l")"
+    pkgs["$pkg"]="$csm"
+  done
+
+  mkdir -p "$bv"
+
+  # Download each package and verify checksum.
+  #
+  for pkg in "${!pkgs[@]}"; do
+
+    pkgf="$bv/$pkg"
+
+    if [ -f "$pkgf" ]; then
+      error "destination package $pkgf from $bk already exist"
+    fi
+
+    info "fetching $url/$pkg"
+    curl -sSf -o "$pkgf" "$url/$pkg"
+
+    csm="$(sha256sum -b "$pkgf" | sed -rn -e 's#^([^ ]+) \*(.+)$#\1#p')"
+
+    if [ "$csm" != "${pkgs[$pkg]}" ]; then
+      error "checksum mismatch for package $pkgf from $bk"
+    fi
+
+    # Overlay the odb windows archive with the contents of the MinGW GCC
+    # distribution archive (see odb/manifest for details).
+    #
+    odb_dir="$(sed -rn -e 's/^(odb-.+-x86_64-windows.+)\.zip$/\1/p' <<<"$pkg")"
+
+    if [ -n "$odb_dir" ]; then
+
+      if [ -d "$odb_dir" ]; then
+        error "package directory $odb_dir already exist"
+      fi
+
+      mingw_file="$(sed -rn -e 's#^.+/([^/]+)$#\1#p' <<<"$mingw")"
+      mingw_dir="$(sed -rn -e 's/^(.+)\.[^.]+\.[^.]+$/\1/p' <<<"$mingw_file")"
+
+      if [ -d "$mingw_dir" ]; then
+        error "MinGW GCC distribution directory $mingw_dir already exist"
+      fi
+
+      info "fetching $mingw"
+      curl -sSf -o "$mingw_file" "$mingw"
+
+      info "overlaying $pkgf with $mingw_file"
+
+      unzip -q "$pkgf"
+      rm "$pkgf"
+
+      tar -xf "$mingw_file"
+      rm "$mingw_file"
+
+      mv "$mingw_dir" "$odb_dir/mingw"
+      cp "$odb_dir/mingw/bin/libgcc_s_seh-"*.dll "$odb_dir/bin"
+      cp "$odb_dir/mingw/bin/libstdc++-"*.dll "$odb_dir/bin"
+      cp "$odb_dir/mingw/bin/libwinpthread-"*.dll "$odb_dir/bin"
+
+      zip -9 -rq "$pkgf" "$odb_dir"
+      rm -r "$odb_dir"
+    fi
+  done
+
+  dirs["$bv"]=true
+done
+fi
+
+# Generate packages.sha256
+#
+for d in "${!dirs[@]}"; do
+  cd "$d"
+  rm -f "packages.sha256"
+  l="$(find . -maxdepth 1 -type f -printf '%P\n' | sort)" # Array-like.
+  sha256sum -b $l >"packages.sha256"
+  cd "$owd"
+done