path: root/build2/download

#! /usr/bin/env bash

# Download and arrange source and binary packages.
#
# Also combine the xsde-<version>-x86_64-linux*.tar.xz and
# xsde-<version>-x86_64-windows*.zip binary archives with the libxsde and
# xsde-{tests,examples} source archives and the dist directory of the xsde git
# repository into the xsde-<version>-{x86_64-linux,rtsrc-unix}.tar.xz and
# xsde-<version>-x86_64-windows.zip archives with the secondary build system,
# respectively.
#
# Run from the root of the destination download directory. Note that the
# destination files are expected not to exist.

repo="https://queue.stage.build2.org"

bin_repo="$repo/0/bindist"
src_repo="$repo/1"

dist_dir="$HOME/work/xsde/xsde/dist"

# Optionally, override any source files in the resulting archives (can be
# helpful during the xsde development). Key is file path relative to the above
# dist directory which, presumably, belongs to the being developed xsde git
# project. Value is file path in the resulting archives relative to the top
# directory of their contents.
#
declare -A overrides
#overrides["../libxsde/xsde/cxx/config.hxx"]="libxsde/xsde/cxx/config.hxx"

declare -A vers
declare -A srcs
declare -A bins

# Project to version map.
#
# Key is project name, value is space-separated list of the
# '[<src-repo-url>/]<repo-section>/<version>' entries. For entries with the
# omitted '<src-repo-url>/' prefix the "$src_repo/" prefix is assumed. For
# example:
#
# alpha/3.3.1-a.0.20241206125527.c3e2855e0a52 https://pkg.cppget.org/1/stable/1.1.1+21
#
# Note that for the binary distribution packages only the first version is
# used.
#
vers["xsde"]="alpha/3.4.0-b.1.20250205115531.5a363116081f"

# Package to project map.
#
# Key is package name, value is project name (for all specified versions).
#
srcs["xsde"]="xsde"
srcs["xsde-tests"]="xsde"
srcs["xsde-examples"]="xsde"
srcs["libxsde"]="xsde"

# Binary source to destination path map.
#
# The key is the "<dist>/<os>/<proj>/<pkg>/<cfg>" bindist artifact path. The
# value is the destination subdirectory.
#
# Note that <cfg> is a sanitized package configuration name prefixed with an
# architecture (see brep's upload-bindist handler for details).
#
bins["debian/debian11/xsde/xsde/x86_64"]="debian/debian11/x86_64"
bins["debian/debian11/xsde/libxsde/x86_64"]="debian/debian11/x86_64"

bins["debian/debian12/xsde/xsde/x86_64"]="debian/debian12/x86_64"
bins["debian/debian12/xsde/libxsde/x86_64"]="debian/debian12/x86_64"

bins["debian/ubuntu22.04/xsde/xsde/x86_64"]="ubuntu/ubuntu22.04/x86_64"
bins["debian/ubuntu22.04/xsde/libxsde/x86_64"]="ubuntu/ubuntu22.04/x86_64"

bins["debian/ubuntu24.04/xsde/xsde/x86_64"]="ubuntu/ubuntu24.04/x86_64"
bins["debian/ubuntu24.04/xsde/libxsde/x86_64"]="ubuntu/ubuntu24.04/x86_64"

bins["fedora/fedora40/xsde/xsde/x86_64"]="fedora/fedora40/x86_64"
bins["fedora/fedora40/xsde/libxsde/x86_64"]="fedora/fedora40/x86_64"

bins["fedora/fedora41/xsde/xsde/x86_64"]="fedora/fedora41/x86_64"
bins["fedora/fedora41/xsde/libxsde/x86_64"]="fedora/fedora41/x86_64"

bins["fedora/rhel9.2/xsde/xsde/x86_64"]="rhel/rhel9.2/x86_64"
bins["fedora/rhel9.2/xsde/libxsde/x86_64"]="rhel/rhel9.2/x86_64"

bins["archive/debian11/xsde/xsde/x86_64-linux-glibc2.31"]="linux/linux-glibc2.31/x86_64"
bins["archive/debian11/xsde/libxsde/x86_64-linux"]="linux/linux-glibc2.31/x86_64"

bins["archive/windows10/xsde/xsde/x86_64"]="windows/windows10/x86_64"
bins["archive/windows10/xsde/libxsde/x86_64"]="windows/windows10/x86_64"

owd=`pwd`
trap "{ cd $owd; exit 1; }" ERR
set -o errtrace   # Trap in functions.
set -o pipefail   # Fail if any pipeline command fails.
shopt -s lastpipe # Execute last pipeline command in current shell.

function info () { echo "$*" 1>&2; }
function error () { info "$*"; exit 1; }

declare -A dirs # Package directories to generate packages.sha256 in.

# Download and arrange sources.
#
# While at it, collect the paths to the source package archives which will be
# used to produce the archives with the secondary build system. Note that only
# the first source package version is used.
#
libxsde_arch=
tests_arch=
examples_arch=

if true; then
for pkg in "${!srcs[@]}"; do
  prj="${srcs[$pkg]}"

  for v in ${vers["$prj"]}; do
    ver="$(sed -rn -e 's#^.+/([^/]+)$#\1#p' <<<"$v")"
    sec="$(sed -rn -e 's#^(.+/)?([^/]+)/[^/]+$#\2#p' <<<"$v")"
    rep="$(sed -rn -e 's#^((.+)/)?[^/]+/[^/]+$#\2#p' <<<"$v")"

    if [ -z "$rep" ]; then
      rep="$src_repo"
    fi

    pkgf="$pkg-$ver.tar.gz"
    url="$rep/$sec/$prj/$pkgf"

    info "fetching $url"
    curl -sSf -o "$pkgf" "$url"

    if [ "$pkg" == "libxsde" -a -z "$libxsde_arch" ]; then
      libxsde_arch="$pkgf"
    elif [ "$pkg" == "xsde-tests" -a -z "$tests_arch" ]; then
      tests_arch="$pkgf"
    elif [ "$pkg" == "xsde-examples" -a -z "$examples_arch" ]; then
      examples_arch="$pkgf"
    fi
  done
done
dirs["."]=true
fi

# Download and arrange binaries.
#

# Create an archive with the secondary build system. The <bin_dir>, <doc_dir>,
# and <man_dir> paths are relative to the top directory of the original xsde
# binary archive contents (usr/local/bin, etc). If <bin_dir> is empty, then
# the binary files are not copied and the resulting archive is a purely source
# archive. If the <windows> argument is true, then remove the install-sh
# script and add the '.txt' extension to the text files without extension
# (INSTALL, README, etc) and convert them to the DOS format (add CR prior to
# LF). Otherwise, remove all the nmakefiles.
#
# Note that the top directory of each archive is expected to be named in the
# same way as the archive itself with the extension stripped.
#
# <dst_arch> <xsde_arch> <bin_dir> <doc_dir> <man_dir> <windows>
#
function create_arch ()
{
  local dst_arch="$1"
  local xsde_arch="$2"
  local bin_dir="$3"
  local doc_dir="$4"
  local man_dir="$5"
  local windows="$6"

  info "combining $xsde_arch with $libxsde_arch, $tests_arch, $examples_arch, and $dist_dir/ into $dst_arch"

  if [ -f "$dst_arch" ]; then
    error "destination package $dst_arch already exist"
  fi

  # Verify and print the extension of the specified archive path. The valid
  # extensions are tar.gz, tar.xz, and zip. Fail for invalid extensions.
  #
  function arch_ext () # <arc_path>
  {
    local r
    r="$(sed -rn -e 's/^.+\.(tar\.(xz|gz)|zip)$/\1/p' <<<"$1")"

    if [ -z "$r" ]; then
      error "unexpected extension for package archive '$1'"
    fi

    echo "$r"
  }

  # Extract content of the specified archive.
  #
  function extract () # <arc_path>
  {
    if [ "$(arch_ext "$1")" = "zip" ]; then
      unzip -q "$1"
    else # tar.xz, tar.gz
      tar -xf "$1"
    fi
  }

  local dst_dir
  dst_dir="$(basename "$dst_arch" ".$(arch_ext "$dst_arch")")"

  if [ -d "$dst_dir" ]; then
    error "package directory $dst_dir already exist"
  fi

  # Add the contents of the xsde binary archive.
  #
  local src_dir
  src_dir="$(basename "$xsde_arch" ".$(arch_ext "$xsde_arch")")"

  if [ -d "$src_dir" ]; then
    error "package directory $src_dir already exist"
  fi

  extract "$xsde_arch"

  if [ ! -d "$src_dir" ]; then
    error "package archive $xsde_arch doesn't contain directory $src_dir"
  fi

  mkdir -p "$dst_dir/doc"

  if [ -n "$bin_dir" ]; then
    mv "$src_dir/$bin_dir" "$dst_dir"
  fi

  mv "$src_dir/$doc_dir/"{cxx,xsde.xhtml,default.css} "$dst_dir/doc"
  mv "$src_dir/$man_dir/xsde.1" "$dst_dir/doc"

  rm -r "$src_dir"

  # Add the contents of the libxsde source archive.
  #
  src_dir="$(basename "$libxsde_arch" ".$(arch_ext "$libxsde_arch")")"

  if [ -d "$src_dir" ]; then
    error "package directory $src_dir already exist"
  fi

  extract "$libxsde_arch"

  if [ ! -d "$src_dir" ]; then
    error "package archive $libxsde_arch doesn't contain directory $src_dir"
  fi

  mkdir "$dst_dir/libxsde"

  mv "$src_dir/xsde" "$dst_dir/libxsde"
  mv "$src_dir/"{FLOSSE,GPLv2,LICENSE,NEWS} "$dst_dir"

  find "$dst_dir/libxsde" \( -name buildfile -o -name '*.in' \) -delete

  rm -r "$src_dir"

  # Add the contents of the xsde-tests source archive.
  #
  src_dir="$(basename "$tests_arch" ".$(arch_ext "$tests_arch")")"

  if [ -d "$src_dir" ]; then
    error "package directory $src_dir already exist"
  fi

  extract "$tests_arch"

  if [ ! -d "$src_dir" ]; then
    error "package archive $tests_arch doesn't contain directory $src_dir"
  fi

  mkdir "$dst_dir/tests"

  mv "$src_dir/cxx" "$dst_dir/tests"

  find "$dst_dir/tests" \( -name buildfile -o -name testscript \) -delete

  rm -r "$src_dir"

  # Add the contents of the xsde-examples source archive.
  #
  src_dir="$(basename "$examples_arch" ".$(arch_ext "$examples_arch")")"

  if [ -d "$src_dir" ]; then
    error "package directory $src_dir already exist"
  fi

  extract "$examples_arch"

  if [ ! -d "$src_dir" ]; then
    error "package archive $examples_arch doesn't contain directory $src_dir"
  fi

  mkdir "$dst_dir/examples"

  mv "$src_dir/cxx" "$dst_dir/examples"

  find "$dst_dir/examples" \( -name buildfile -o -name testscript \) -delete

  rm -r "$src_dir"

  # Add the contents of the dist directory.
  #
  cp -rp "$dist_dir"/* "$dst_dir"

  # Override any specified files from the source archives with their local
  # versions.
  #
  local f
  for f in "${!overrides[@]}"; do
    cp "$dist_dir/$f" "$dst_dir/${overrides[$f]}"
  done

  # Perform the Windows/POSIX-related cleanups.
  #
  if [[ "$windows" ]]; then
    # Remove install-sh script.
    #
    rm "$dst_dir/install-sh"

    # Rename and convert the text files.
    #
    local f
    for f in "$dst_dir/"{FLOSSE,GPLv2,INSTALL,LICENSE,NEWS}; do
      todos -ep "$f"
      mv "$f" "$f.txt"
    done

    find "$dst_dir" -name README -exec todos -ep "{}" \; -exec mv "{}" "{}.txt" \;
  else
    # Remove nmakefiles. If etc/*/ subdirectory contains nmakefiles, then
    # remove the whole subdirectory.
    #
    local f
    for f in $(find "$dst_dir/etc" \( -name nmakefile -o -name '*.nmake' \) -execdir pwd \;); do
      rm -rf "$f"
    done

    find "$dst_dir" \( -name nmakefile -o -name '*.nmake' \) -delete
  fi

  # Create the resulting archive.
  #
  case "$(arch_ext "$dst_arch")" in
    "zip")
      zip -9 -rq "$dst_arch" "$dst_dir"
      ;;

    "tar.xz")
      tar -cf - "$dst_dir" | xz -9 >"$dst_arch"
      ;;

    "tar.gz")
      tar -cf - "$dst_dir" | gzip -9 >"$dst_arch"
      ;;
  esac

  rm -r "$dst_dir"
}

if true; then
# If the xsde-<version>-rtsrc-unix.tar.xz archive is created (in the current
# directory), then also create symlinks to it in all subdirectories where the
# binary xsde packages/archives are downloaded to or created in, except for
# the Windows archives.
#
xsde_rtsrc_unix_arch=

for bk in "${!bins[@]}"; do
  bv="${bins[$bk]}"

  # Get the distribution type we are dealing with.
  #
  dist="$(sed -rn -e 's#^(.+)/.+/.+/.+/.+$#\1#p' <<<"$bk")"

  # Get the project we are dealing with and its version. Pick the first
  # version from potentially multiple ones specified for the project.
  #
  prj="$(sed -rn -e 's#^.+/.+/(.+)/.+/.+$#\1#p' <<<"$bk")"

  v="$(sed -rn -e 's#^([^ ]+).*$#\1#p' <<<"${vers[$prj]}")"
  ver="$(sed -rn -e 's#^.+/([^/]+)$#\1#p' <<<"$v")"

  url="$bin_repo/"
  url+="$(sed -rn -e "s#^(.+/.+/.+/.+)/(.+)\$#\\1/$ver/\\2#p" <<<"$bk")"

  # Get the list of packages (keys) and their checksums (values) at this URL
  # by loading its packages.sha256.
  #
  unset pkgs # Clear.
  declare -A pkgs

  info "fetching $url/packages.sha256"
  curl -sSf "$url/packages.sha256" | while read l; do

    pkg="$(sed -rn -e 's#^([^ ]+) \*(.+)$#\2#p' <<<"$l")"

    # Get the extension and omit certain distribution-specific file types.
    #
    e="$(sed -rn -e 's#^.+\.([^.]+)$#\1#p' <<<"$pkg")"

    case "$dist" in
      "debian")
        if [ "$e" = "buildinfo" -o "$e" = "changes" ]; then
          continue
        fi
	      ;;

      "fedora")
        ;;
    esac

    csm="$(sed -rn -e 's#^([^ ]+) \*(.+)$#\1#p' <<<"$l")"
    pkgs["$pkg"]="$csm"
  done

  mkdir -p "$bv"

  # Download each package and verify checksum.
  #
  for pkg in "${!pkgs[@]}"; do

    pkgf="$bv/$pkg"

    if [ -f "$pkgf" ]; then
      error "destination package $pkgf from $bk already exist"
    fi

    info "fetching $url/$pkg"
    curl -sSf -o "$pkgf" "$url/$pkg"

    csm="$(sha256sum -b "$pkgf" | sed -rn -e 's#^([^ ]+) \*(.+)$#\1#p')"

    if [ "$csm" != "${pkgs[$pkg]}" ]; then
      error "checksum mismatch for package $pkgf from $bk"
    fi

    # Create the xsde-<version>-{x86_64-linux,rtsrc-unix}.tar.xz archives,
    # combining the contents of the xsde binary linux archive with the
    # contents of the libxsde, xsde-tests, and xsde-examples source archives
    # and the dist directory.
    #
    if [ -n "$(sed -rn -e 's/^xsde-.+-x86_64-linux.+\.tar\.xz$/true/p' <<<"$pkg")" ]; then
      dest_arch="$(sed -rn -e 's%^(.+/xsde-[^/]+-x86_64)-[^/]+$%\1-linux.tar.xz%p' <<<"$pkgf")"

      create_arch "$dest_arch" "$pkgf" \
                  usr/local/bin \
                  usr/local/share/doc/xsde \
                  usr/local/share/man/man1 \
                  "" # windows

      # Note that we don't include the architecture into the resulting archive
      # name since this is a source archive. Also note that we create it in
      # the current directory.
      #
      xsde_rtsrc_unix_arch="$(sed -rn -e 's%^.+/(xsde-[^/]+)-x86_64-[^/]+$%\1-rtsrc-unix.tar.xz%p' <<<"$pkgf")"

      create_arch "$xsde_rtsrc_unix_arch" "$pkgf" \
                  "" \
                  usr/local/share/doc/xsde \
                  usr/local/share/man/man1 \
                  "" # windows

      rm -r "$pkgf"
    fi

    # Create the xsde-<version>-x86_64-windows.zip archive, similar to the
    # above.
    #
    if [ -n "$(sed -rn -e 's/^xsde-.+-x86_64-windows.+\.zip$/true/p' <<<"$pkg")" ]; then
      dest_arch="$(sed -rn -e 's%^(.+/xsde-[^/]+-x86_64)-[^/]+$%\1-windows.zip%p' <<<"$pkgf")"

      create_arch "$dest_arch" "$pkgf" \
                  bin \
                  share/doc/xsde \
                  share/man/man1 \
                  true # windows

      rm -r "$pkgf"
    fi
  done

  dirs["$bv"]=true
done

# Create symlinks to xsde-<version>-rtsrc-unix.tar.xz, if present.
#
if [ -n "$xsde_rtsrc_unix_arch" ]; then
  for bk in "${!bins[@]}"; do
    pkg="$(sed -rn -e 's#^.+/.+/.+/(.+)/.+$#\1#p' <<<"$bk")"
    win="$(sed -rn -e 's#^.+/(windows.*)/.+/.+/.+$#true#p' <<<"$bk")"

    if [ "$pkg" == "xsde" -a -z "$win" ]; then
      bv="${bins[$bk]}"

      ln -s "$(sed -r -e 's/[^/]+/../g' <<<"$bv")/$xsde_rtsrc_unix_arch" \
            "$bv/$xsde_rtsrc_unix_arch"
    fi
  done
fi
fi

# Generate packages.sha256
#
for d in "${!dirs[@]}"; do
  cd "$d"
  rm -f packages.sha256
  l="$(find -L . -maxdepth 1 -type f -printf '%P\n' | sort)" # Array-like.
  sha256sum -b $l >packages.sha256
  cd "$owd"
done