path: root/build2/download

#! /usr/bin/env bash

# Download and arrange source and binary packages.
#
# Run from the root of the destination download directory. Note that the
# destination files are expected not to exist.

repo="https://queue.stage.build2.org"
#mingw="https://download.build2.org/0.17.0/build2-mingw-0.17.0-x86_64-windows.tar.xz"
mingw="https://stage.build2.org/0/0.18.0-a.0/build2-mingw-0.18.0-a.0-x86_64-windows.tar.xz"

bin_repo="$repo/0/bindist"
src_repo="$repo/1"

declare -A vers
declare -A srcs
declare -A bins

# Project to version map.
#
# Key is project name, value is space-separated list of the
# '[<src-repo-url>/]<repo-section>/<version>' entries. For entries with the
# omitted '<src-repo-url>/' prefix the "$src_repo/" prefix is assumed. For
# example:
#
# alpha/3.3.1-a.0.20241206125527.c3e2855e0a52 https://pkg.cppget.org/1/stable/1.1.1+21
#
# Note that for the binary distribution packages only the first version is
# used.
#
# Note that currently we download two versions of openssl libraries since
# libmysqlclient 8.0.15+18 constrains the libcrypto and libssl dependency
# versions with ^1.1.1.
#
vers["odb"]="testing/2.5.0"
vers["mysql"]="https://pkg.cppget.org/1/stable/8.0.15+18"
vers["sqlite"]="https://pkg.cppget.org/1/stable/3.45.3+1"
vers["postgresql"]="https://pkg.cppget.org/1/stable/14.1.0+6"
vers["libcutl"]="https://pkg.cppget.org/1/stable/1.11.0"
vers["libstudxml"]="https://pkg.cppget.org/1/testing/1.1.0"
vers["zlib"]="https://pkg.cppget.org/1/stable/1.2.1200+5"
vers["zstd"]="https://pkg.cppget.org/1/stable/1.5.5+1"
vers["openssl"]="https://pkg.cppget.org/1/stable/3.3.1 https://pkg.cppget.org/1/stable/1.1.1+21"
vers["isocline"]="https://pkg.cppget.org/1/stable/1.0.9+1"

# Package to project map.
#
# Key is package name, value is project name (for all specified versions).
#
# Note that we don't download the Boost and Qt source packages, which may
# potentially be required for building odb-tests and odb-examples, since there
# are too large.
#
srcs["libodb"]="odb"
srcs["libodb-mysql"]="odb"
srcs["libodb-sqlite"]="odb"
srcs["libodb-pgsql"]="odb"
srcs["libodb-oracle"]="odb"
srcs["libodb-mssql"]="odb"
srcs["libodb-boost"]="odb"
srcs["libodb-qt"]="odb"
srcs["odb"]="odb"
srcs["odb-tests"]="odb"
srcs["odb-examples"]="odb"
srcs["libmysqlclient"]="mysql"
srcs["mysql-client"]="mysql"
srcs["libsqlite3"]="sqlite"
srcs["sqlite3"]="sqlite"
srcs["libpq"]="postgresql"
srcs["psql"]="postgresql"
srcs["libcutl"]="libcutl"
srcs["libstudxml"]="libstudxml"
srcs["libz"]="zlib"
srcs["libzstd"]="zstd"
srcs["libcrypto"]="openssl"
srcs["libssl"]="openssl"
srcs["libisocline"]="isocline"

# Binary source to destination path map.
#
# The key is the "<dist>/<os>/<proj>/<pkg>/<cfg>" bindist artifact path. The
# value is the destination subdirectory.
#
# Note that <cfg> is a sanitized package configuration name prefixed with an
# architecture (see brep's upload-bindist handler for details).
#
bins["debian/debian11/odb/libodb/x86_64"]="debian/debian11/x86_64"
bins["debian/debian11/odb/libodb-mysql/x86_64"]="debian/debian11/x86_64"
bins["debian/debian11/odb/libodb-sqlite/x86_64"]="debian/debian11/x86_64"
bins["debian/debian11/odb/libodb-pgsql/x86_64"]="debian/debian11/x86_64"
bins["debian/debian11/odb/libodb-boost/x86_64"]="debian/debian11/x86_64"
bins["debian/debian11/odb/libodb-qt/x86_64"]="debian/debian11/x86_64"
bins["debian/debian11/odb/odb/x86_64"]="debian/debian11/x86_64"

bins["debian/debian12/odb/libodb/x86_64"]="debian/debian12/x86_64"
bins["debian/debian12/odb/libodb-mysql/x86_64"]="debian/debian12/x86_64"
bins["debian/debian12/odb/libodb-sqlite/x86_64"]="debian/debian12/x86_64"
bins["debian/debian12/odb/libodb-pgsql/x86_64"]="debian/debian12/x86_64"
bins["debian/debian12/odb/libodb-boost/x86_64"]="debian/debian12/x86_64"
bins["debian/debian12/odb/libodb-qt/x86_64"]="debian/debian12/x86_64"
bins["debian/debian12/odb/odb/x86_64"]="debian/debian12/x86_64"

bins["debian/ubuntu22.04/odb/libodb/x86_64"]="ubuntu/ubuntu22.04/x86_64"
bins["debian/ubuntu22.04/odb/libodb-mysql/x86_64"]="ubuntu/ubuntu22.04/x86_64"
bins["debian/ubuntu22.04/odb/libodb-sqlite/x86_64"]="ubuntu/ubuntu22.04/x86_64"
bins["debian/ubuntu22.04/odb/libodb-pgsql/x86_64"]="ubuntu/ubuntu22.04/x86_64"
bins["debian/ubuntu22.04/odb/libodb-boost/x86_64"]="ubuntu/ubuntu22.04/x86_64"
bins["debian/ubuntu22.04/odb/libodb-qt/x86_64"]="ubuntu/ubuntu22.04/x86_64"
bins["debian/ubuntu22.04/odb/odb/x86_64"]="ubuntu/ubuntu22.04/x86_64"

bins["debian/ubuntu24.04/odb/libodb/x86_64"]="ubuntu/ubuntu24.04/x86_64"
bins["debian/ubuntu24.04/odb/libodb-mysql/x86_64"]="ubuntu/ubuntu24.04/x86_64"
bins["debian/ubuntu24.04/odb/libodb-sqlite/x86_64"]="ubuntu/ubuntu24.04/x86_64"
bins["debian/ubuntu24.04/odb/libodb-pgsql/x86_64"]="ubuntu/ubuntu24.04/x86_64"
bins["debian/ubuntu24.04/odb/libodb-boost/x86_64"]="ubuntu/ubuntu24.04/x86_64"
bins["debian/ubuntu24.04/odb/libodb-qt/x86_64"]="ubuntu/ubuntu24.04/x86_64"
bins["debian/ubuntu24.04/odb/odb/x86_64"]="ubuntu/ubuntu24.04/x86_64"

bins["fedora/fedora40/odb/libodb/x86_64"]="fedora/fedora40/x86_64"
bins["fedora/fedora40/odb/libodb-mysql/x86_64"]="fedora/fedora40/x86_64"
bins["fedora/fedora40/odb/libodb-sqlite/x86_64"]="fedora/fedora40/x86_64"
bins["fedora/fedora40/odb/libodb-pgsql/x86_64"]="fedora/fedora40/x86_64"
bins["fedora/fedora40/odb/libodb-boost/x86_64"]="fedora/fedora40/x86_64"
bins["fedora/fedora40/odb/libodb-qt/x86_64"]="fedora/fedora40/x86_64"
bins["fedora/fedora40/odb/odb/x86_64"]="fedora/fedora40/x86_64"

bins["fedora/fedora41/odb/libodb/x86_64"]="fedora/fedora41/x86_64"
bins["fedora/fedora41/odb/libodb-mysql/x86_64"]="fedora/fedora41/x86_64"
bins["fedora/fedora41/odb/libodb-sqlite/x86_64"]="fedora/fedora41/x86_64"
bins["fedora/fedora41/odb/libodb-pgsql/x86_64"]="fedora/fedora41/x86_64"
bins["fedora/fedora41/odb/libodb-boost/x86_64"]="fedora/fedora41/x86_64"
bins["fedora/fedora41/odb/libodb-qt/x86_64"]="fedora/fedora41/x86_64"
bins["fedora/fedora41/odb/odb/x86_64"]="fedora/fedora41/x86_64"

bins["fedora/rhel9.2/odb/libodb/x86_64"]="rhel/rhel9.2/x86_64"
bins["fedora/rhel9.2/odb/libodb-mysql/x86_64"]="rhel/rhel9.2/x86_64"
bins["fedora/rhel9.2/odb/libodb-sqlite/x86_64"]="rhel/rhel9.2/x86_64"
bins["fedora/rhel9.2/odb/libodb-pgsql/x86_64"]="rhel/rhel9.2/x86_64"
bins["fedora/rhel9.2/odb/libodb-boost/x86_64"]="rhel/rhel9.2/x86_64"
bins["fedora/rhel9.2/odb/libodb-qt/x86_64"]="rhel/rhel9.2/x86_64"
bins["fedora/rhel9.2/odb/odb/x86_64"]="rhel/rhel9.2/x86_64"

bins["archive/windows10/odb/libodb/x86_64-release"]="windows/windows10/x86_64"
bins["archive/windows10/odb/libodb/x86_64-debug"]="windows/windows10/x86_64"
bins["archive/windows10/odb/libodb-mysql/x86_64-release"]="windows/windows10/x86_64"
bins["archive/windows10/odb/libodb-mysql/x86_64-debug"]="windows/windows10/x86_64"
bins["archive/windows10/odb/libodb-sqlite/x86_64-release"]="windows/windows10/x86_64"
bins["archive/windows10/odb/libodb-sqlite/x86_64-debug"]="windows/windows10/x86_64"
bins["archive/windows10/odb/libodb-pgsql/x86_64-release"]="windows/windows10/x86_64"
bins["archive/windows10/odb/libodb-pgsql/x86_64-debug"]="windows/windows10/x86_64"
bins["archive/windows10/odb/libodb-boost/x86_64-release"]="windows/windows10/x86_64"
bins["archive/windows10/odb/libodb-boost/x86_64-debug"]="windows/windows10/x86_64"
bins["archive/windows10/odb/libodb-qt/x86_64-release"]="windows/windows10/x86_64"
bins["archive/windows10/odb/libodb-qt/x86_64-debug"]="windows/windows10/x86_64"
bins["archive/windows10/odb/odb/x86_64"]="windows/windows10/x86_64"

owd=`pwd`
trap "{ cd $owd; exit 1; }" ERR
set -o errtrace   # Trap in functions.
set -o pipefail   # Fail if any pipeline command fails.
shopt -s lastpipe # Execute last pipeline command in current shell.

function info () { echo "$*" 1>&2; }
function error () { info "$*"; exit 1; }

declare -A dirs # Package directories to generate packages.sha256 in.

# Download and arrange sources.
#
if true; then
for pkg in "${!srcs[@]}"; do
  prj="${srcs[$pkg]}"

  for v in ${vers["$prj"]}; do

    ver="$(sed -rn -e 's#^.+/([^/]+)$#\1#p' <<<"$v")"
    sec="$(sed -rn -e 's#^(.+/)?([^/]+)/[^/]+$#\2#p' <<<"$v")"
    rep="$(sed -rn -e 's#^((.+)/)?[^/]+/[^/]+$#\2#p' <<<"$v")"

    if [ -z "$rep" ]; then
      rep="$src_repo"
    fi

    pkgf="$pkg-$ver.tar.gz"
    url="$rep/$sec/$prj/$pkgf"

    info "fetching $url"
    curl -sSf -o "$pkgf" "$url"

    # Make .zip archive for examples.
    #
    if [ -n "$(sed -rn -e 's/^.+-examples$/true/p' <<<"$pkg")" ]; then
      pkgd="$pkg-$ver"

      if [ -d "$pkgd" ]; then
        error "package directory $pkgd already exist"
      fi

      info "repacking $pkgf"
      tar -xf "$pkgf"
      zip -9 -rq "$pkg-$ver.zip" "$pkgd"
      rm -rf "$pkgd"
    fi
  done
done
dirs["."]=true
fi

# Download and arrange binaries.
#
# Also overlay MinGW distribution over the ODB compiler archive for Windows.
#
if true; then
for bk in "${!bins[@]}"; do
  bv="${bins[$bk]}"

  # Get the distribution type we are dealing with.
  #
  dist="$(sed -rn -e 's#^(.+)/.+/.+/.+/.+$#\1#p' <<<"$bk")"

  # Get the project we are dealing with and its version. Pick the first
  # version from potentially multiple ones specified for the project.
  #
  prj="$(sed -rn -e 's#^.+/.+/(.+)/.+/.+$#\1#p' <<<"$bk")"

  v="$(sed -rn -e 's#^([^ ]+).*$#\1#p' <<<"${vers[$prj]}")"
  ver="$(sed -rn -e 's#^.+/([^/]+)$#\1#p' <<<"$v")"

  url="$bin_repo/"
  url+="$(sed -rn -e "s#^(.+/.+/.+/.+)/(.+)\$#\\1/$ver/\\2#p" <<<"$bk")"

  # Get the list of packages (keys) and their checksums (values) at this URL
  # by loading its packages.sha256.
  #
  unset pkgs # Clear.
  declare -A pkgs

  info "fetching $url/packages.sha256"
  curl -sSf "$url/packages.sha256" | while read l; do

    pkg="$(sed -rn -e 's#^([^ ]+) \*(.+)$#\2#p' <<<"$l")"

    # Get the extension and omit certain distribution-specific file types.
    #
    e="$(sed -rn -e 's#^.+\.([^.]+)$#\1#p' <<<"$pkg")"

    case "$dist" in
      "debian")
        if [ "$e" = "buildinfo" -o "$e" = "changes" ]; then
          continue
        fi
	      ;;

      "fedora")
        ;;
    esac

    csm="$(sed -rn -e 's#^([^ ]+) \*(.+)$#\1#p' <<<"$l")"
    pkgs["$pkg"]="$csm"
  done

  mkdir -p "$bv"

  # Download each package and verify checksum.
  #
  for pkg in "${!pkgs[@]}"; do

    pkgf="$bv/$pkg"

    if [ -f "$pkgf" ]; then
      error "destination package $pkgf from $bk already exist"
    fi

    info "fetching $url/$pkg"
    curl -sSf -o "$pkgf" "$url/$pkg"

    csm="$(sha256sum -b "$pkgf" | sed -rn -e 's#^([^ ]+) \*(.+)$#\1#p')"

    if [ "$csm" != "${pkgs[$pkg]}" ]; then
      error "checksum mismatch for package $pkgf from $bk"
    fi

    # Overlay the odb windows archive with the contents of the MinGW GCC
    # distribution archive (see odb/manifest for details).
    #
    odb_dir="$(sed -rn -e 's/^(odb-.+-x86_64-windows.+)\.zip$/\1/p' <<<"$pkg")"

    if [ -n "$odb_dir" ]; then

      if [ -d "$odb_dir" ]; then
        error "package directory $odb_dir already exist"
      fi

      mingw_file="$(sed -rn -e 's#^.+/([^/]+)$#\1#p' <<<"$mingw")"
      mingw_dir="$(sed -rn -e 's/^(.+)\.[^.]+\.[^.]+$/\1/p' <<<"$mingw_file")"

      if [ -d "$mingw_dir" ]; then
        error "MinGW GCC distribution directory $mingw_dir already exist"
      fi

      info "fetching $mingw"
      curl -sSf -o "$mingw_file" "$mingw"

      info "overlaying $pkgf with $mingw_file"

      unzip -q "$pkgf"
      rm "$pkgf"

      tar -xf "$mingw_file"
      rm "$mingw_file"

      mv "$mingw_dir" "$odb_dir/mingw"
      cp "$odb_dir/mingw/bin/libgcc_s_seh-"*.dll "$odb_dir/bin"
      cp "$odb_dir/mingw/bin/libstdc++-"*.dll "$odb_dir/bin"
      cp "$odb_dir/mingw/bin/libwinpthread-"*.dll "$odb_dir/bin"

      zip -9 -rq "$pkgf" "$odb_dir"
      rm -r "$odb_dir"
    fi
  done

  dirs["$bv"]=true
done
fi

# Generate packages.sha256
#
for d in "${!dirs[@]}"; do
  cd "$d"
  rm -f "packages.sha256"
  l="$(find . -maxdepth 1 -type f -printf '%P\n' | sort)" # Array-like.
  sha256sum -b $l >"packages.sha256"
  cd "$owd"
done