Add build2/download script

1 files changed, 503 insertions, 0 deletions

diff --git a/build2/download b/build2/download
new file mode 100755
index 0000000..20081d3
--- /dev/null
+++ b/build2/download
@@ -0,0 +1,503 @@
+#! /usr/bin/env bash
+
+# Download and arrange source and binary packages.
+#
+# Also combine the xsde-<version>-x86_64-linux*.tar.xz and
+# xsde-<version>-x86_64-windows*.zip binary archives with the libxsde and
+# xsde-{tests,examples} source archives and the dist directory of the xsde git
+# repository into the xsde-<version>-{x86_64-linux,rtsrc-unix}.tar.xz and
+# xsde-<version>-x86_64-windows.zip archives with the secondary build system,
+# respectively.
+#
+# Run from the root of the destination download directory. Note that the
+# destination files are expected not to exist.
+
+repo="https://queue.stage.build2.org"
+
+bin_repo="$repo/0/bindist"
+src_repo="$repo/1"
+
+dist_dir="$HOME/work/xsde/xsde/dist"
+
+# Optionally, override any source files in the resulting archives (can be
+# helpful during the xsde development). Key is file path relative to the above
+# dist directory which, presumably, belongs to the being developed xsde git
+# project. Value is file path in the resulting archives relative to the top
+# directory of their contents.
+#
+declare -A overrides
+#overrides["../libxsde/xsde/cxx/config.hxx"]="libxsde/xsde/cxx/config.hxx"
+
+declare -A vers
+declare -A srcs
+declare -A bins
+
+# Project to version map.
+#
+# Key is project name, value is space-separated list of the
+# '[<src-repo-url>/]<repo-section>/<version>' entries. For entries with the
+# omitted '<src-repo-url>/' prefix the "$src_repo/" prefix is assumed. For
+# example:
+#
+# alpha/3.3.1-a.0.20241206125527.c3e2855e0a52 https://pkg.cppget.org/1/stable/1.1.1+21
+#
+# Note that for the binary distribution packages only the first version is
+# used.
+#
+vers["xsde"]="alpha/3.4.0-b.1.20250205115531.5a363116081f"
+
+# Package to project map.
+#
+# Key is package name, value is project name (for all specified versions).
+#
+srcs["xsde"]="xsde"
+srcs["xsde-tests"]="xsde"
+srcs["xsde-examples"]="xsde"
+srcs["libxsde"]="xsde"
+
+# Binary source to destination path map.
+#
+# The key is the "<dist>/<os>/<proj>/<pkg>/<cfg>" bindist artifact path. The
+# value is the destination subdirectory.
+#
+# Note that <cfg> is a sanitized package configuration name prefixed with an
+# architecture (see brep's upload-bindist handler for details).
+#
+bins["debian/debian11/xsde/xsde/x86_64"]="debian/debian11/x86_64"
+bins["debian/debian11/xsde/libxsde/x86_64"]="debian/debian11/x86_64"
+
+bins["debian/debian12/xsde/xsde/x86_64"]="debian/debian12/x86_64"
+bins["debian/debian12/xsde/libxsde/x86_64"]="debian/debian12/x86_64"
+
+bins["debian/ubuntu22.04/xsde/xsde/x86_64"]="ubuntu/ubuntu22.04/x86_64"
+bins["debian/ubuntu22.04/xsde/libxsde/x86_64"]="ubuntu/ubuntu22.04/x86_64"
+
+bins["debian/ubuntu24.04/xsde/xsde/x86_64"]="ubuntu/ubuntu24.04/x86_64"
+bins["debian/ubuntu24.04/xsde/libxsde/x86_64"]="ubuntu/ubuntu24.04/x86_64"
+
+bins["fedora/fedora40/xsde/xsde/x86_64"]="fedora/fedora40/x86_64"
+bins["fedora/fedora40/xsde/libxsde/x86_64"]="fedora/fedora40/x86_64"
+
+bins["fedora/fedora41/xsde/xsde/x86_64"]="fedora/fedora41/x86_64"
+bins["fedora/fedora41/xsde/libxsde/x86_64"]="fedora/fedora41/x86_64"
+
+bins["fedora/rhel9.2/xsde/xsde/x86_64"]="rhel/rhel9.2/x86_64"
+bins["fedora/rhel9.2/xsde/libxsde/x86_64"]="rhel/rhel9.2/x86_64"
+
+bins["archive/debian11/xsde/xsde/x86_64-linux-glibc2.31"]="linux/linux-glibc2.31/x86_64"
+bins["archive/debian11/xsde/libxsde/x86_64-linux"]="linux/linux-glibc2.31/x86_64"
+
+bins["archive/windows10/xsde/xsde/x86_64"]="windows/windows10/x86_64"
+bins["archive/windows10/xsde/libxsde/x86_64"]="windows/windows10/x86_64"
+
+owd=`pwd`
+trap "{ cd $owd; exit 1; }" ERR
+set -o errtrace   # Trap in functions.
+set -o pipefail   # Fail if any pipeline command fails.
+shopt -s lastpipe # Execute last pipeline command in current shell.
+
+function info () { echo "$*" 1>&2; }
+function error () { info "$*"; exit 1; }
+
+declare -A dirs # Package directories to generate packages.sha256 in.
+
+# Download and arrange sources.
+#
+# While at it, collect the paths to the source package archives which will be
+# used to produce the archives with the secondary build system. Note that only
+# the first source package version is used.
+#
+libxsde_arch=
+tests_arch=
+examples_arch=
+
+if true; then
+for pkg in "${!srcs[@]}"; do
+  prj="${srcs[$pkg]}"
+
+  for v in ${vers["$prj"]}; do
+    ver="$(sed -rn -e 's#^.+/([^/]+)$#\1#p' <<<"$v")"
+    sec="$(sed -rn -e 's#^(.+/)?([^/]+)/[^/]+$#\2#p' <<<"$v")"
+    rep="$(sed -rn -e 's#^((.+)/)?[^/]+/[^/]+$#\2#p' <<<"$v")"
+
+    if [ -z "$rep" ]; then
+      rep="$src_repo"
+    fi
+
+    pkgf="$pkg-$ver.tar.gz"
+    url="$rep/$sec/$prj/$pkgf"
+
+    info "fetching $url"
+    curl -sSf -o "$pkgf" "$url"
+
+    if [ "$pkg" == "libxsde" -a -z "$libxsde_arch" ]; then
+      libxsde_arch="$pkgf"
+    elif [ "$pkg" == "xsde-tests" -a -z "$tests_arch" ]; then
+      tests_arch="$pkgf"
+    elif [ "$pkg" == "xsde-examples" -a -z "$examples_arch" ]; then
+      examples_arch="$pkgf"
+    fi
+
+    # Make .zip archive for examples.
+    #
+    if [ -n "$(sed -rn -e 's/^.+-examples$/true/p' <<<"$pkg")" ]; then
+      pkgd="$pkg-$ver"
+
+      if [ -d "$pkgd" ]; then
+        error "package directory $pkgd already exist"
+      fi
+
+      info "repacking $pkgf"
+      tar -xf "$pkgf"
+      zip -9 -rq "$pkg-$ver.zip" "$pkgd"
+      rm -rf "$pkgd"
+    fi
+  done
+done
+dirs["."]=true
+fi
+
+# Download and arrange binaries.
+#
+
+# Create an archive with the secondary build system. The <bin_dir>, <doc_dir>,
+# and <man_dir> paths are relative to the top directory of the original xsde
+# binary archive contents (usr/local/bin, etc). If <bin_dir> is empty, then
+# the binary files are not copied and the resulting archive is a purely source
+# archive. If the <windows> argument is true, then remove the install-sh
+# script and add the '.txt' extension to the text files without extension
+# (INSTALL, README, etc) and convert them to the DOS format (add CR prior to
+# LF). Otherwise, remove all the nmakefiles.
+#
+# Note that the top directory of each archive is expected to be named in the
+# same way as the archive itself with the extension stripped.
+#
+# <dst_arch> <xsde_arch> <bin_dir> <doc_dir> <man_dir> <windows>
+#
+function create_arch ()
+{
+  local dst_arch="$1"
+  local xsde_arch="$2"
+  local bin_dir="$3"
+  local doc_dir="$4"
+  local man_dir="$5"
+  local windows="$6"
+
+  info "combining $xsde_arch with $libxsde_arch, $tests_arch, $examples_arch, and $dist_dir/ into $dst_arch"
+
+  if [ -f "$dst_arch" ]; then
+    error "destination package $dst_arch already exist"
+  fi
+
+  # Verify and print the extension of the specified archive path. The valid
+  # extensions are tar.gz, tar.xz, and zip. Fail for invalid extensions.
+  #
+  function arch_ext () # <arc_path>
+  {
+    local r
+    r="$(sed -rn -e 's/^.+\.(tar\.(xz|gz)|zip)$/\1/p' <<<"$1")"
+
+    if [ -z "$r" ]; then
+      error "unexpected extension for package archive '$1'"
+    fi
+
+    echo "$r"
+  }
+
+  # Extract content of the specified archive.
+  #
+  function extract () # <arc_path>
+  {
+    if [ "$(arch_ext "$1")" = "zip" ]; then
+      unzip -q "$1"
+    else # tar.xz, tar.gz
+      tar -xf "$1"
+    fi
+  }
+
+  local dst_dir
+  dst_dir="$(basename "$dst_arch" ".$(arch_ext "$dst_arch")")"
+
+  if [ -d "$dst_dir" ]; then
+    error "package directory $dst_dir already exist"
+  fi
+
+  # Add the contents of the xsde binary archive.
+  #
+  local src_dir
+  src_dir="$(basename "$xsde_arch" ".$(arch_ext "$xsde_arch")")"
+
+  if [ -d "$src_dir" ]; then
+    error "package directory $src_dir already exist"
+  fi
+
+  extract "$xsde_arch"
+
+  if [ ! -d "$src_dir" ]; then
+    error "package archive $xsde_arch doesn't contain directory $src_dir"
+  fi
+
+  mkdir -p "$dst_dir/doc"
+
+  if [ -n "$bin_dir" ]; then
+    mv "$src_dir/$bin_dir" "$dst_dir"
+  fi
+
+  mv "$src_dir/$doc_dir/"{cxx,xsde.xhtml,default.css} "$dst_dir/doc"
+  mv "$src_dir/$man_dir/xsde.1" "$dst_dir/doc"
+
+  rm -r "$src_dir"
+
+  # Add the contents of the libxsde source archive.
+  #
+  src_dir="$(basename "$libxsde_arch" ".$(arch_ext "$libxsde_arch")")"
+
+  if [ -d "$src_dir" ]; then
+    error "package directory $src_dir already exist"
+  fi
+
+  extract "$libxsde_arch"
+
+  if [ ! -d "$src_dir" ]; then
+    error "package archive $libxsde_arch doesn't contain directory $src_dir"
+  fi
+
+  mkdir "$dst_dir/libxsde"
+
+  mv "$src_dir/xsde" "$dst_dir/libxsde"
+  mv "$src_dir/"{FLOSSE,GPLv2,LICENSE,NEWS} "$dst_dir"
+
+  find "$dst_dir/libxsde" \( -name buildfile -o -name '*.in' \) -delete
+
+  rm -r "$src_dir"
+
+  # Add the contents of the xsde-tests source archive.
+  #
+  src_dir="$(basename "$tests_arch" ".$(arch_ext "$tests_arch")")"
+
+  if [ -d "$src_dir" ]; then
+    error "package directory $src_dir already exist"
+  fi
+
+  extract "$tests_arch"
+
+  if [ ! -d "$src_dir" ]; then
+    error "package archive $tests_arch doesn't contain directory $src_dir"
+  fi
+
+  mkdir "$dst_dir/tests"
+
+  mv "$src_dir/cxx" "$dst_dir/tests"
+
+  find "$dst_dir/tests" \( -name buildfile -o -name testscript \) -delete
+
+  rm -r "$src_dir"
+
+  # Add the contents of the xsde-examples source archive.
+  #
+  src_dir="$(basename "$examples_arch" ".$(arch_ext "$examples_arch")")"
+
+  if [ -d "$src_dir" ]; then
+    error "package directory $src_dir already exist"
+  fi
+
+  extract "$examples_arch"
+
+  if [ ! -d "$src_dir" ]; then
+    error "package archive $examples_arch doesn't contain directory $src_dir"
+  fi
+
+  mkdir "$dst_dir/examples"
+
+  mv "$src_dir/cxx" "$dst_dir/examples"
+
+  find "$dst_dir/examples" \( -name buildfile -o -name testscript \) -delete
+
+  rm -r "$src_dir"
+
+  # Add the contents of the dist directory.
+  #
+  cp -rp "$dist_dir"/* "$dst_dir"
+
+  # Override any specified files from the source archives with their local
+  # versions.
+  #
+  local f
+  for f in "${!overrides[@]}"; do
+    cp "$dist_dir/$f" "$dst_dir/${overrides[$f]}"
+  done
+
+  # Perform the Windows/POSIX-related cleanups.
+  #
+  if [[ "$windows" ]]; then
+    # Remove install-sh script.
+    #
+    rm "$dst_dir/install-sh"
+
+    # Rename and convert the text files.
+    #
+    local f
+    for f in "$dst_dir/"{FLOSSE,GPLv2,INSTALL,LICENSE,NEWS}; do
+      todos -ep "$f"
+      mv "$f" "$f.txt"
+    done
+
+    find "$dst_dir" -name README -exec todos -ep "{}" \; -exec mv "{}" "{}.txt" \;
+  else
+    # Remove nmakefiles. If etc/*/ subdirectory contains nmakefiles, then
+    # remove the whole subdirectory.
+    #
+    local f
+    for f in $(find "$dst_dir/etc" \( -name nmakefile -o -name '*.nmake' \) -execdir pwd \;); do
+      rm -rf "$f"
+    done
+
+    find "$dst_dir" \( -name nmakefile -o -name '*.nmake' \) -delete
+  fi
+
+  # Create the resulting archive.
+  #
+  case "$(arch_ext "$dst_arch")" in
+    "zip")
+      zip -9 -rq "$dst_arch" "$dst_dir"
+      ;;
+
+    "tar.xz")
+      tar -cf - "$dst_dir" | xz -9 >"$dst_arch"
+      ;;
+
+    "tar.gz")
+      tar -cf - "$dst_dir" | gzip -9 >"$dst_arch"
+      ;;
+  esac
+
+  rm -r "$dst_dir"
+}
+
+if true; then
+for bk in "${!bins[@]}"; do
+  bv="${bins[$bk]}"
+
+  # Get the distribution type we are dealing with.
+  #
+  dist="$(sed -rn -e 's#^(.+)/.+/.+/.+/.+$#\1#p' <<<"$bk")"
+
+  # Get the project we are dealing with and its version. Pick the first
+  # version from potentially multiple ones specified for the project.
+  #
+  prj="$(sed -rn -e 's#^.+/.+/(.+)/.+/.+$#\1#p' <<<"$bk")"
+
+  v="$(sed -rn -e 's#^([^ ]+).*$#\1#p' <<<"${vers[$prj]}")"
+  ver="$(sed -rn -e 's#^.+/([^/]+)$#\1#p' <<<"$v")"
+
+  url="$bin_repo/"
+  url+="$(sed -rn -e "s#^(.+/.+/.+/.+)/(.+)\$#\\1/$ver/\\2#p" <<<"$bk")"
+
+  # Get the list of packages (keys) and their checksums (values) at this URL
+  # by loading its packages.sha256.
+  #
+  unset pkgs # Clear.
+  declare -A pkgs
+
+  info "fetching $url/packages.sha256"
+  curl -sSf "$url/packages.sha256" | while read l; do
+
+    pkg="$(sed -rn -e 's#^([^ ]+) \*(.+)$#\2#p' <<<"$l")"
+
+    # Get the extension and omit certain distribution-specific file types.
+    #
+    e="$(sed -rn -e 's#^.+\.([^.]+)$#\1#p' <<<"$pkg")"
+
+    case "$dist" in
+      "debian")
+        if [ "$e" = "buildinfo" -o "$e" = "changes" ]; then
+          continue
+        fi
+	      ;;
+
+      "fedora")
+        ;;
+    esac
+
+    csm="$(sed -rn -e 's#^([^ ]+) \*(.+)$#\1#p' <<<"$l")"
+    pkgs["$pkg"]="$csm"
+  done
+
+  mkdir -p "$bv"
+
+  # Download each package and verify checksum.
+  #
+  for pkg in "${!pkgs[@]}"; do
+
+    pkgf="$bv/$pkg"
+
+    if [ -f "$pkgf" ]; then
+      error "destination package $pkgf from $bk already exist"
+    fi
+
+    info "fetching $url/$pkg"
+    curl -sSf -o "$pkgf" "$url/$pkg"
+
+    csm="$(sha256sum -b "$pkgf" | sed -rn -e 's#^([^ ]+) \*(.+)$#\1#p')"
+
+    if [ "$csm" != "${pkgs[$pkg]}" ]; then
+      error "checksum mismatch for package $pkgf from $bk"
+    fi
+
+    # Create the xsde-<version>-{x86_64-linux,rtsrc-unix}.tar.xz archives,
+    # combining the contents of the xsde binary linux archive with the
+    # contents of the libxsde, xsde-tests, and xsde-examples source archives
+    # and the dist directory.
+    #
+    if [ -n "$(sed -rn -e 's/^xsde-.+-x86_64-linux.+\.tar\.xz$/true/p' <<<"$pkg")" ]; then
+      dest_arch="$(sed -rn -e 's%^(.+/xsde-[^/]+-x86_64)-[^/]+$%\1-linux.tar.xz%p' <<<"$pkgf")"
+
+      create_arch "$dest_arch" "$pkgf" \
+                  usr/local/bin \
+                  usr/local/share/doc/xsde \
+                  usr/local/share/man/man1 \
+                  "" # windows
+
+      # Note that we don't include the architecture into the resulting archive
+      # name since this is a source archive.
+      #
+      dest_arch="$(sed -rn -e 's%^(.+/xsde-[^/]+)-x86_64-[^/]+$%\1-rtsrc-unix.tar.xz%p' <<<"$pkgf")"
+
+      create_arch "$dest_arch" "$pkgf" \
+                  "" \
+                  usr/local/share/doc/xsde \
+                  usr/local/share/man/man1 \
+                  "" # windows
+
+      rm -r "$pkgf"
+    fi
+
+    # Create the xsde-<version>-x86_64-windows.zip archive, similar to the
+    # above.
+    #
+    if [ -n "$(sed -rn -e 's/^xsde-.+-x86_64-windows.+\.zip$/true/p' <<<"$pkg")" ]; then
+      dest_arch="$(sed -rn -e 's%^(.+/xsde-[^/]+-x86_64)-[^/]+$%\1-windows.zip%p' <<<"$pkgf")"
+
+      create_arch "$dest_arch" "$pkgf" \
+                  bin \
+                  share/doc/xsde \
+                  share/man/man1 \
+                  true # windows
+
+      rm -r "$pkgf"
+    fi
+  done
+
+  dirs["$bv"]=true
+done
+fi
+
+# Generate packages.sha256
+#
+for d in "${!dirs[@]}"; do
+  cd "$d"
+  rm -f "packages.sha256"
+  l="$(find . -maxdepth 1 -type f -printf '%P\n' | sort)" # Array-like.
+  sha256sum -b $l >"packages.sha256"
+  cd "$owd"
+done