diff options
Diffstat (limited to 'build2/download')
| -rwxr-xr-x | build2/download | 334 |
1 files changed, 334 insertions, 0 deletions
diff --git a/build2/download b/build2/download new file mode 100755 index 0000000..8851c12 --- /dev/null +++ b/build2/download @@ -0,0 +1,334 @@ +#! /usr/bin/env bash + +# Download and arrange source and binary packages. +# +# Run from the root of the destination download directory. Note that the +# destination files are expected not to exist. + +repo="https://queue.stage.build2.org" +#mingw="https://download.build2.org/0.17.0/build2-mingw-0.17.0-x86_64-windows.tar.xz" +mingw="https://stage.build2.org/0/0.18.0-a.0/build2-mingw-0.18.0-a.0-x86_64-windows.tar.xz" + +bin_repo="$repo/0/bindist" +src_repo="$repo/1" + +declare -A vers +declare -A srcs +declare -A bins + +# Project to version map. +# +# Key is project name, value is space-separated list of the +# '[<src-repo-url>/]<repo-section>/<version>' entries. For entries with the +# omitted '<src-repo-url>/' prefix the "$src_repo/" prefix is assumed. For +# example: +# +# alpha/3.3.1-a.0.20241206125527.c3e2855e0a52 https://pkg.cppget.org/1/stable/1.1.1+21 +# +# Note that for the binary distribution packages only the first version is +# used. +# +# Note that currently we download two versions of openssl libraries since +# libmysqlclient 8.0.15+18 constrains the libcrypto and libssl dependency +# versions with ^1.1.1. +# +vers["odb"]="testing/2.5.0" +vers["mysql"]="https://pkg.cppget.org/1/stable/8.0.15+18" +vers["sqlite"]="https://pkg.cppget.org/1/stable/3.45.3+1" +vers["postgresql"]="https://pkg.cppget.org/1/stable/14.1.0+6" +vers["libcutl"]="https://pkg.cppget.org/1/stable/1.11.0" +vers["libstudxml"]="https://pkg.cppget.org/1/testing/1.1.0" +vers["zlib"]="https://pkg.cppget.org/1/stable/1.2.1200+5" +vers["zstd"]="https://pkg.cppget.org/1/stable/1.5.5+1" +vers["openssl"]="https://pkg.cppget.org/1/stable/3.3.1 https://pkg.cppget.org/1/stable/1.1.1+21" +vers["isocline"]="https://pkg.cppget.org/1/stable/1.0.9+1" + +# Package to project map. +# +# Key is package name, value is project name (for all specified versions). +# +# Note that we don't download the Boost and Qt source packages, which may +# potentially be required for building odb-tests and odb-examples, since there +# are too large. +# +srcs["libodb"]="odb" +srcs["libodb-mysql"]="odb" +srcs["libodb-sqlite"]="odb" +srcs["libodb-pgsql"]="odb" +srcs["libodb-oracle"]="odb" +srcs["libodb-mssql"]="odb" +srcs["libodb-boost"]="odb" +srcs["libodb-qt"]="odb" +srcs["odb"]="odb" +srcs["odb-tests"]="odb" +srcs["odb-examples"]="odb" +srcs["libmysqlclient"]="mysql" +srcs["mysql-client"]="mysql" +srcs["libsqlite3"]="sqlite" +srcs["sqlite3"]="sqlite" +srcs["libpq"]="postgresql" +srcs["psql"]="postgresql" +srcs["libcutl"]="libcutl" +srcs["libstudxml"]="libstudxml" +srcs["libz"]="zlib" +srcs["libzstd"]="zstd" +srcs["libcrypto"]="openssl" +srcs["libssl"]="openssl" +srcs["libisocline"]="isocline" + +# Binary source to destination path map. +# +# The key is the "<dist>/<os>/<proj>/<pkg>/<cfg>" bindist artifact path. The +# value is the destination subdirectory. +# +# Note that <cfg> is a sanitized package configuration name prefixed with an +# architecture (see brep's upload-bindist handler for details). +# +bins["debian/debian11/odb/libodb/x86_64"]="debian/debian11/x86_64" +bins["debian/debian11/odb/libodb-mysql/x86_64"]="debian/debian11/x86_64" +bins["debian/debian11/odb/libodb-sqlite/x86_64"]="debian/debian11/x86_64" +bins["debian/debian11/odb/libodb-pgsql/x86_64"]="debian/debian11/x86_64" +bins["debian/debian11/odb/libodb-boost/x86_64"]="debian/debian11/x86_64" +bins["debian/debian11/odb/libodb-qt/x86_64"]="debian/debian11/x86_64" +bins["debian/debian11/odb/odb/x86_64"]="debian/debian11/x86_64" + +bins["debian/debian12/odb/libodb/x86_64"]="debian/debian12/x86_64" +bins["debian/debian12/odb/libodb-mysql/x86_64"]="debian/debian12/x86_64" +bins["debian/debian12/odb/libodb-sqlite/x86_64"]="debian/debian12/x86_64" +bins["debian/debian12/odb/libodb-pgsql/x86_64"]="debian/debian12/x86_64" +bins["debian/debian12/odb/libodb-boost/x86_64"]="debian/debian12/x86_64" +bins["debian/debian12/odb/libodb-qt/x86_64"]="debian/debian12/x86_64" +bins["debian/debian12/odb/odb/x86_64"]="debian/debian12/x86_64" + +bins["debian/ubuntu22.04/odb/libodb/x86_64"]="ubuntu/ubuntu22.04/x86_64" +bins["debian/ubuntu22.04/odb/libodb-mysql/x86_64"]="ubuntu/ubuntu22.04/x86_64" +bins["debian/ubuntu22.04/odb/libodb-sqlite/x86_64"]="ubuntu/ubuntu22.04/x86_64" +bins["debian/ubuntu22.04/odb/libodb-pgsql/x86_64"]="ubuntu/ubuntu22.04/x86_64" +bins["debian/ubuntu22.04/odb/libodb-boost/x86_64"]="ubuntu/ubuntu22.04/x86_64" +bins["debian/ubuntu22.04/odb/libodb-qt/x86_64"]="ubuntu/ubuntu22.04/x86_64" +bins["debian/ubuntu22.04/odb/odb/x86_64"]="ubuntu/ubuntu22.04/x86_64" + +bins["debian/ubuntu24.04/odb/libodb/x86_64"]="ubuntu/ubuntu24.04/x86_64" +bins["debian/ubuntu24.04/odb/libodb-mysql/x86_64"]="ubuntu/ubuntu24.04/x86_64" +bins["debian/ubuntu24.04/odb/libodb-sqlite/x86_64"]="ubuntu/ubuntu24.04/x86_64" +bins["debian/ubuntu24.04/odb/libodb-pgsql/x86_64"]="ubuntu/ubuntu24.04/x86_64" +bins["debian/ubuntu24.04/odb/libodb-boost/x86_64"]="ubuntu/ubuntu24.04/x86_64" +bins["debian/ubuntu24.04/odb/libodb-qt/x86_64"]="ubuntu/ubuntu24.04/x86_64" +bins["debian/ubuntu24.04/odb/odb/x86_64"]="ubuntu/ubuntu24.04/x86_64" + +bins["fedora/fedora40/odb/libodb/x86_64"]="fedora/fedora40/x86_64" +bins["fedora/fedora40/odb/libodb-mysql/x86_64"]="fedora/fedora40/x86_64" +bins["fedora/fedora40/odb/libodb-sqlite/x86_64"]="fedora/fedora40/x86_64" +bins["fedora/fedora40/odb/libodb-pgsql/x86_64"]="fedora/fedora40/x86_64" +bins["fedora/fedora40/odb/libodb-boost/x86_64"]="fedora/fedora40/x86_64" +bins["fedora/fedora40/odb/libodb-qt/x86_64"]="fedora/fedora40/x86_64" +bins["fedora/fedora40/odb/odb/x86_64"]="fedora/fedora40/x86_64" + +bins["fedora/fedora41/odb/libodb/x86_64"]="fedora/fedora41/x86_64" +bins["fedora/fedora41/odb/libodb-mysql/x86_64"]="fedora/fedora41/x86_64" +bins["fedora/fedora41/odb/libodb-sqlite/x86_64"]="fedora/fedora41/x86_64" +bins["fedora/fedora41/odb/libodb-pgsql/x86_64"]="fedora/fedora41/x86_64" +bins["fedora/fedora41/odb/libodb-boost/x86_64"]="fedora/fedora41/x86_64" +bins["fedora/fedora41/odb/libodb-qt/x86_64"]="fedora/fedora41/x86_64" +bins["fedora/fedora41/odb/odb/x86_64"]="fedora/fedora41/x86_64" + +bins["fedora/rhel9.2/odb/libodb/x86_64"]="rhel/rhel9.2/x86_64" +bins["fedora/rhel9.2/odb/libodb-mysql/x86_64"]="rhel/rhel9.2/x86_64" +bins["fedora/rhel9.2/odb/libodb-sqlite/x86_64"]="rhel/rhel9.2/x86_64" +bins["fedora/rhel9.2/odb/libodb-pgsql/x86_64"]="rhel/rhel9.2/x86_64" +bins["fedora/rhel9.2/odb/libodb-boost/x86_64"]="rhel/rhel9.2/x86_64" +bins["fedora/rhel9.2/odb/libodb-qt/x86_64"]="rhel/rhel9.2/x86_64" +bins["fedora/rhel9.2/odb/odb/x86_64"]="rhel/rhel9.2/x86_64" + +bins["archive/windows10/odb/libodb/x86_64-release"]="windows/windows10/x86_64" +bins["archive/windows10/odb/libodb/x86_64-debug"]="windows/windows10/x86_64" +bins["archive/windows10/odb/libodb-mysql/x86_64-release"]="windows/windows10/x86_64" +bins["archive/windows10/odb/libodb-mysql/x86_64-debug"]="windows/windows10/x86_64" +bins["archive/windows10/odb/libodb-sqlite/x86_64-release"]="windows/windows10/x86_64" +bins["archive/windows10/odb/libodb-sqlite/x86_64-debug"]="windows/windows10/x86_64" +bins["archive/windows10/odb/libodb-pgsql/x86_64-release"]="windows/windows10/x86_64" +bins["archive/windows10/odb/libodb-pgsql/x86_64-debug"]="windows/windows10/x86_64" +bins["archive/windows10/odb/libodb-boost/x86_64-release"]="windows/windows10/x86_64" +bins["archive/windows10/odb/libodb-boost/x86_64-debug"]="windows/windows10/x86_64" +bins["archive/windows10/odb/libodb-qt/x86_64-release"]="windows/windows10/x86_64" +bins["archive/windows10/odb/libodb-qt/x86_64-debug"]="windows/windows10/x86_64" +bins["archive/windows10/odb/odb/x86_64"]="windows/windows10/x86_64" + +owd=`pwd` +trap "{ cd $owd; exit 1; }" ERR +set -o errtrace # Trap in functions. +set -o pipefail # Fail if any pipeline command fails. +shopt -s lastpipe # Execute last pipeline command in current shell. + +function info () { echo "$*" 1>&2; } +function error () { info "$*"; exit 1; } + +declare -A dirs # Package directories to generate packages.sha256 in. + +# Download and arrange sources. +# +if true; then +for pkg in "${!srcs[@]}"; do + prj="${srcs[$pkg]}" + + for v in ${vers["$prj"]}; do + + ver="$(sed -rn -e 's#^.+/([^/]+)$#\1#p' <<<"$v")" + sec="$(sed -rn -e 's#^(.+/)?([^/]+)/[^/]+$#\2#p' <<<"$v")" + rep="$(sed -rn -e 's#^((.+)/)?[^/]+/[^/]+$#\2#p' <<<"$v")" + + if [ -z "$rep" ]; then + rep="$src_repo" + fi + + pkgf="$pkg-$ver.tar.gz" + url="$rep/$sec/$prj/$pkgf" + + info "fetching $url" + curl -sSf -o "$pkgf" "$url" + + # Make .zip archive for examples. + # + if [ -n "$(sed -rn -e 's/^.+-examples$/true/p' <<<"$pkg")" ]; then + pkgd="$pkg-$ver" + + if [ -d "$pkgd" ]; then + error "package directory $pkgd already exist" + fi + + info "repacking $pkgf" + tar -xf "$pkgf" + zip -9 -rq "$pkg-$ver.zip" "$pkgd" + rm -rf "$pkgd" + fi + done +done +dirs["."]=true +fi + +# Download and arrange binaries. +# +# Also overlay MinGW distribution over the ODB compiler archive for Windows. +# +if true; then +for bk in "${!bins[@]}"; do + bv="${bins[$bk]}" + + # Get the distribution type we are dealing with. + # + dist="$(sed -rn -e 's#^(.+)/.+/.+/.+/.+$#\1#p' <<<"$bk")" + + # Get the project we are dealing with and its version. Pick the first + # version from potentially multiple ones specified for the project. + # + prj="$(sed -rn -e 's#^.+/.+/(.+)/.+/.+$#\1#p' <<<"$bk")" + + v="$(sed -rn -e 's#^([^ ]+).*$#\1#p' <<<"${vers[$prj]}")" + ver="$(sed -rn -e 's#^.+/([^/]+)$#\1#p' <<<"$v")" + + url="$bin_repo/" + url+="$(sed -rn -e "s#^(.+/.+/.+/.+)/(.+)\$#\\1/$ver/\\2#p" <<<"$bk")" + + # Get the list of packages (keys) and their checksums (values) at this URL + # by loading its packages.sha256. + # + unset pkgs # Clear. + declare -A pkgs + + info "fetching $url/packages.sha256" + curl -sSf "$url/packages.sha256" | while read l; do + + pkg="$(sed -rn -e 's#^([^ ]+) \*(.+)$#\2#p' <<<"$l")" + + # Get the extension and omit certain distribution-specific file types. + # + e="$(sed -rn -e 's#^.+\.([^.]+)$#\1#p' <<<"$pkg")" + + case "$dist" in + "debian") + if [ "$e" = "buildinfo" -o "$e" = "changes" ]; then + continue + fi + ;; + + "fedora") + ;; + esac + + csm="$(sed -rn -e 's#^([^ ]+) \*(.+)$#\1#p' <<<"$l")" + pkgs["$pkg"]="$csm" + done + + mkdir -p "$bv" + + # Download each package and verify checksum. + # + for pkg in "${!pkgs[@]}"; do + + pkgf="$bv/$pkg" + + if [ -f "$pkgf" ]; then + error "destination package $pkgf from $bk already exist" + fi + + info "fetching $url/$pkg" + curl -sSf -o "$pkgf" "$url/$pkg" + + csm="$(sha256sum -b "$pkgf" | sed -rn -e 's#^([^ ]+) \*(.+)$#\1#p')" + + if [ "$csm" != "${pkgs[$pkg]}" ]; then + error "checksum mismatch for package $pkgf from $bk" + fi + + # Overlay the odb windows archive with the contents of the MinGW GCC + # distribution archive (see odb/manifest for details). + # + odb_dir="$(sed -rn -e 's/^(odb-.+-x86_64-windows.+)\.zip$/\1/p' <<<"$pkg")" + + if [ -n "$odb_dir" ]; then + + if [ -d "$odb_dir" ]; then + error "package directory $odb_dir already exist" + fi + + mingw_file="$(sed -rn -e 's#^.+/([^/]+)$#\1#p' <<<"$mingw")" + mingw_dir="$(sed -rn -e 's/^(.+)\.[^.]+\.[^.]+$/\1/p' <<<"$mingw_file")" + + if [ -d "$mingw_dir" ]; then + error "MinGW GCC distribution directory $mingw_dir already exist" + fi + + info "fetching $mingw" + curl -sSf -o "$mingw_file" "$mingw" + + info "overlaying $pkgf with $mingw_file" + + unzip -q "$pkgf" + rm "$pkgf" + + tar -xf "$mingw_file" + rm "$mingw_file" + + mv "$mingw_dir" "$odb_dir/mingw" + cp "$odb_dir/mingw/bin/libgcc_s_seh-"*.dll "$odb_dir/bin" + cp "$odb_dir/mingw/bin/libstdc++-"*.dll "$odb_dir/bin" + cp "$odb_dir/mingw/bin/libwinpthread-"*.dll "$odb_dir/bin" + + zip -9 -rq "$pkgf" "$odb_dir" + rm -r "$odb_dir" + fi + done + + dirs["$bv"]=true +done +fi + +# Generate packages.sha256 +# +for d in "${!dirs[@]}"; do + cd "$d" + rm -f "packages.sha256" + l="$(find . -maxdepth 1 -type f -printf '%P\n' | sort)" # Array-like. + sha256sum -b $l >"packages.sha256" + cd "$owd" +done |