#! /usr/bin/env bash # Download and arrange source and binary packages. # # Also combine the xsde--x86_64-linux*.tar.xz and # xsde--x86_64-windows*.zip binary archives with the libxsde and # xsde-{tests,examples} source archives and the dist directory of the xsde git # repository into the xsde--{x86_64-linux,rtsrc-unix}.tar.xz and # xsde--x86_64-windows.zip archives with the secondary build system, # respectively. # # Run from the root of the destination download directory. Note that the # destination files are expected not to exist. repo="https://queue.stage.build2.org" bin_repo="$repo/0/bindist" src_repo="$repo/1" dist_dir="$HOME/work/xsde/xsde/dist" # Optionally, override any source files in the resulting archives (can be # helpful during the xsde development). Key is file path relative to the above # dist directory which, presumably, belongs to the being developed xsde git # project. Value is file path in the resulting archives relative to the top # directory of their contents. # declare -A overrides #overrides["../libxsde/xsde/cxx/config.hxx"]="libxsde/xsde/cxx/config.hxx" declare -A vers declare -A srcs declare -A bins # Project to version map. # # Key is project name, value is space-separated list of the # '[/]/' entries. For entries with the # omitted '/' prefix the "$src_repo/" prefix is assumed. For # example: # # alpha/3.3.1-a.0.20241206125527.c3e2855e0a52 https://pkg.cppget.org/1/stable/1.1.1+21 # # Note that for the binary distribution packages only the first version is # used. # vers["xsde"]="alpha/3.4.0-b.1.20250205115531.5a363116081f" # Package to project map. # # Key is package name, value is project name (for all specified versions). # srcs["xsde"]="xsde" srcs["xsde-tests"]="xsde" srcs["xsde-examples"]="xsde" srcs["libxsde"]="xsde" # Binary source to destination path map. # # The key is the "////" bindist artifact path. The # value is the destination subdirectory. # # Note that is a sanitized package configuration name prefixed with an # architecture (see brep's upload-bindist handler for details). # bins["debian/debian11/xsde/xsde/x86_64"]="debian/debian11/x86_64" bins["debian/debian11/xsde/libxsde/x86_64"]="debian/debian11/x86_64" bins["debian/debian12/xsde/xsde/x86_64"]="debian/debian12/x86_64" bins["debian/debian12/xsde/libxsde/x86_64"]="debian/debian12/x86_64" bins["debian/ubuntu22.04/xsde/xsde/x86_64"]="ubuntu/ubuntu22.04/x86_64" bins["debian/ubuntu22.04/xsde/libxsde/x86_64"]="ubuntu/ubuntu22.04/x86_64" bins["debian/ubuntu24.04/xsde/xsde/x86_64"]="ubuntu/ubuntu24.04/x86_64" bins["debian/ubuntu24.04/xsde/libxsde/x86_64"]="ubuntu/ubuntu24.04/x86_64" bins["fedora/fedora40/xsde/xsde/x86_64"]="fedora/fedora40/x86_64" bins["fedora/fedora40/xsde/libxsde/x86_64"]="fedora/fedora40/x86_64" bins["fedora/fedora41/xsde/xsde/x86_64"]="fedora/fedora41/x86_64" bins["fedora/fedora41/xsde/libxsde/x86_64"]="fedora/fedora41/x86_64" bins["fedora/rhel9.2/xsde/xsde/x86_64"]="rhel/rhel9.2/x86_64" bins["fedora/rhel9.2/xsde/libxsde/x86_64"]="rhel/rhel9.2/x86_64" bins["archive/debian11/xsde/xsde/x86_64-linux-glibc2.31"]="linux/linux-glibc2.31/x86_64" bins["archive/debian11/xsde/libxsde/x86_64-linux"]="linux/linux-glibc2.31/x86_64" bins["archive/windows10/xsde/xsde/x86_64"]="windows/windows10/x86_64" bins["archive/windows10/xsde/libxsde/x86_64"]="windows/windows10/x86_64" owd=`pwd` trap "{ cd $owd; exit 1; }" ERR set -o errtrace # Trap in functions. set -o pipefail # Fail if any pipeline command fails. shopt -s lastpipe # Execute last pipeline command in current shell. function info () { echo "$*" 1>&2; } function error () { info "$*"; exit 1; } declare -A dirs # Package directories to generate packages.sha256 in. # Download and arrange sources. # # While at it, collect the paths to the source package archives which will be # used to produce the archives with the secondary build system. Note that only # the first source package version is used. # libxsde_arch= tests_arch= examples_arch= if true; then for pkg in "${!srcs[@]}"; do prj="${srcs[$pkg]}" for v in ${vers["$prj"]}; do ver="$(sed -rn -e 's#^.+/([^/]+)$#\1#p' <<<"$v")" sec="$(sed -rn -e 's#^(.+/)?([^/]+)/[^/]+$#\2#p' <<<"$v")" rep="$(sed -rn -e 's#^((.+)/)?[^/]+/[^/]+$#\2#p' <<<"$v")" if [ -z "$rep" ]; then rep="$src_repo" fi pkgf="$pkg-$ver.tar.gz" url="$rep/$sec/$prj/$pkgf" info "fetching $url" curl -sSf -o "$pkgf" "$url" if [ "$pkg" == "libxsde" -a -z "$libxsde_arch" ]; then libxsde_arch="$pkgf" elif [ "$pkg" == "xsde-tests" -a -z "$tests_arch" ]; then tests_arch="$pkgf" elif [ "$pkg" == "xsde-examples" -a -z "$examples_arch" ]; then examples_arch="$pkgf" fi done done dirs["."]=true fi # Download and arrange binaries. # # Create an archive with the secondary build system. The , , # and paths are relative to the top directory of the original xsde # binary archive contents (usr/local/bin, etc). If is empty, then # the binary files are not copied and the resulting archive is a purely source # archive. If the argument is true, then remove the install-sh # script and add the '.txt' extension to the text files without extension # (INSTALL, README, etc) and convert them to the DOS format (add CR prior to # LF). Otherwise, remove all the nmakefiles. # # Note that the top directory of each archive is expected to be named in the # same way as the archive itself with the extension stripped. # # # function create_arch () { local dst_arch="$1" local xsde_arch="$2" local bin_dir="$3" local doc_dir="$4" local man_dir="$5" local windows="$6" info "combining $xsde_arch with $libxsde_arch, $tests_arch, $examples_arch, and $dist_dir/ into $dst_arch" if [ -f "$dst_arch" ]; then error "destination package $dst_arch already exist" fi # Verify and print the extension of the specified archive path. The valid # extensions are tar.gz, tar.xz, and zip. Fail for invalid extensions. # function arch_ext () # { local r r="$(sed -rn -e 's/^.+\.(tar\.(xz|gz)|zip)$/\1/p' <<<"$1")" if [ -z "$r" ]; then error "unexpected extension for package archive '$1'" fi echo "$r" } # Extract content of the specified archive. # function extract () # { if [ "$(arch_ext "$1")" = "zip" ]; then unzip -q "$1" else # tar.xz, tar.gz tar -xf "$1" fi } local dst_dir dst_dir="$(basename "$dst_arch" ".$(arch_ext "$dst_arch")")" if [ -d "$dst_dir" ]; then error "package directory $dst_dir already exist" fi # Add the contents of the xsde binary archive. # local src_dir src_dir="$(basename "$xsde_arch" ".$(arch_ext "$xsde_arch")")" if [ -d "$src_dir" ]; then error "package directory $src_dir already exist" fi extract "$xsde_arch" if [ ! -d "$src_dir" ]; then error "package archive $xsde_arch doesn't contain directory $src_dir" fi mkdir -p "$dst_dir/doc" if [ -n "$bin_dir" ]; then mv "$src_dir/$bin_dir" "$dst_dir" fi mv "$src_dir/$doc_dir/"{cxx,xsde.xhtml,default.css} "$dst_dir/doc" mv "$src_dir/$man_dir/xsde.1" "$dst_dir/doc" rm -r "$src_dir" # Add the contents of the libxsde source archive. # src_dir="$(basename "$libxsde_arch" ".$(arch_ext "$libxsde_arch")")" if [ -d "$src_dir" ]; then error "package directory $src_dir already exist" fi extract "$libxsde_arch" if [ ! -d "$src_dir" ]; then error "package archive $libxsde_arch doesn't contain directory $src_dir" fi mkdir "$dst_dir/libxsde" mv "$src_dir/xsde" "$dst_dir/libxsde" mv "$src_dir/"{FLOSSE,GPLv2,LICENSE,NEWS} "$dst_dir" find "$dst_dir/libxsde" \( -name buildfile -o -name '*.in' \) -delete rm -r "$src_dir" # Add the contents of the xsde-tests source archive. # src_dir="$(basename "$tests_arch" ".$(arch_ext "$tests_arch")")" if [ -d "$src_dir" ]; then error "package directory $src_dir already exist" fi extract "$tests_arch" if [ ! -d "$src_dir" ]; then error "package archive $tests_arch doesn't contain directory $src_dir" fi mkdir "$dst_dir/tests" mv "$src_dir/cxx" "$dst_dir/tests" find "$dst_dir/tests" \( -name buildfile -o -name testscript \) -delete rm -r "$src_dir" # Add the contents of the xsde-examples source archive. # src_dir="$(basename "$examples_arch" ".$(arch_ext "$examples_arch")")" if [ -d "$src_dir" ]; then error "package directory $src_dir already exist" fi extract "$examples_arch" if [ ! -d "$src_dir" ]; then error "package archive $examples_arch doesn't contain directory $src_dir" fi mkdir "$dst_dir/examples" mv "$src_dir/cxx" "$dst_dir/examples" find "$dst_dir/examples" \( -name buildfile -o -name testscript \) -delete rm -r "$src_dir" # Add the contents of the dist directory. # cp -rp "$dist_dir"/* "$dst_dir" # Override any specified files from the source archives with their local # versions. # local f for f in "${!overrides[@]}"; do cp "$dist_dir/$f" "$dst_dir/${overrides[$f]}" done # Perform the Windows/POSIX-related cleanups. # if [[ "$windows" ]]; then # Remove install-sh script. # rm "$dst_dir/install-sh" # Rename and convert the text files. # local f for f in "$dst_dir/"{FLOSSE,GPLv2,INSTALL,LICENSE,NEWS}; do todos -ep "$f" mv "$f" "$f.txt" done find "$dst_dir" -name README -exec todos -ep "{}" \; -exec mv "{}" "{}.txt" \; else # Remove nmakefiles. If etc/*/ subdirectory contains nmakefiles, then # remove the whole subdirectory. # local f for f in $(find "$dst_dir/etc" \( -name nmakefile -o -name '*.nmake' \) -execdir pwd \;); do rm -rf "$f" done find "$dst_dir" \( -name nmakefile -o -name '*.nmake' \) -delete fi # Create the resulting archive. # case "$(arch_ext "$dst_arch")" in "zip") zip -9 -rq "$dst_arch" "$dst_dir" ;; "tar.xz") tar -cf - "$dst_dir" | xz -9 >"$dst_arch" ;; "tar.gz") tar -cf - "$dst_dir" | gzip -9 >"$dst_arch" ;; esac rm -r "$dst_dir" } if true; then # If the xsde--rtsrc-unix.tar.xz archive is created (in the current # directory), then also create symlinks to it in all subdirectories where the # binary xsde packages/archives are downloaded to or created in, except for # the Windows archives. # xsde_rtsrc_unix_arch= for bk in "${!bins[@]}"; do bv="${bins[$bk]}" # Get the distribution type we are dealing with. # dist="$(sed -rn -e 's#^(.+)/.+/.+/.+/.+$#\1#p' <<<"$bk")" # Get the project we are dealing with and its version. Pick the first # version from potentially multiple ones specified for the project. # prj="$(sed -rn -e 's#^.+/.+/(.+)/.+/.+$#\1#p' <<<"$bk")" v="$(sed -rn -e 's#^([^ ]+).*$#\1#p' <<<"${vers[$prj]}")" ver="$(sed -rn -e 's#^.+/([^/]+)$#\1#p' <<<"$v")" url="$bin_repo/" url+="$(sed -rn -e "s#^(.+/.+/.+/.+)/(.+)\$#\\1/$ver/\\2#p" <<<"$bk")" # Get the list of packages (keys) and their checksums (values) at this URL # by loading its packages.sha256. # unset pkgs # Clear. declare -A pkgs info "fetching $url/packages.sha256" curl -sSf "$url/packages.sha256" | while read l; do pkg="$(sed -rn -e 's#^([^ ]+) \*(.+)$#\2#p' <<<"$l")" # Get the extension and omit certain distribution-specific file types. # e="$(sed -rn -e 's#^.+\.([^.]+)$#\1#p' <<<"$pkg")" case "$dist" in "debian") if [ "$e" = "buildinfo" -o "$e" = "changes" ]; then continue fi ;; "fedora") ;; esac csm="$(sed -rn -e 's#^([^ ]+) \*(.+)$#\1#p' <<<"$l")" pkgs["$pkg"]="$csm" done mkdir -p "$bv" # Download each package and verify checksum. # for pkg in "${!pkgs[@]}"; do pkgf="$bv/$pkg" if [ -f "$pkgf" ]; then error "destination package $pkgf from $bk already exist" fi info "fetching $url/$pkg" curl -sSf -o "$pkgf" "$url/$pkg" csm="$(sha256sum -b "$pkgf" | sed -rn -e 's#^([^ ]+) \*(.+)$#\1#p')" if [ "$csm" != "${pkgs[$pkg]}" ]; then error "checksum mismatch for package $pkgf from $bk" fi # Create the xsde--{x86_64-linux,rtsrc-unix}.tar.xz archives, # combining the contents of the xsde binary linux archive with the # contents of the libxsde, xsde-tests, and xsde-examples source archives # and the dist directory. # if [ -n "$(sed -rn -e 's/^xsde-.+-x86_64-linux.+\.tar\.xz$/true/p' <<<"$pkg")" ]; then dest_arch="$(sed -rn -e 's%^(.+/xsde-[^/]+-x86_64)-[^/]+$%\1-linux.tar.xz%p' <<<"$pkgf")" create_arch "$dest_arch" "$pkgf" \ usr/local/bin \ usr/local/share/doc/xsde \ usr/local/share/man/man1 \ "" # windows # Note that we don't include the architecture into the resulting archive # name since this is a source archive. Also note that we create it in # the current directory. # xsde_rtsrc_unix_arch="$(sed -rn -e 's%^.+/(xsde-[^/]+)-x86_64-[^/]+$%\1-rtsrc-unix.tar.xz%p' <<<"$pkgf")" create_arch "$xsde_rtsrc_unix_arch" "$pkgf" \ "" \ usr/local/share/doc/xsde \ usr/local/share/man/man1 \ "" # windows rm -r "$pkgf" fi # Create the xsde--x86_64-windows.zip archive, similar to the # above. # if [ -n "$(sed -rn -e 's/^xsde-.+-x86_64-windows.+\.zip$/true/p' <<<"$pkg")" ]; then dest_arch="$(sed -rn -e 's%^(.+/xsde-[^/]+-x86_64)-[^/]+$%\1-windows.zip%p' <<<"$pkgf")" create_arch "$dest_arch" "$pkgf" \ bin \ share/doc/xsde \ share/man/man1 \ true # windows rm -r "$pkgf" fi done dirs["$bv"]=true done # Create symlinks to xsde--rtsrc-unix.tar.xz, if present. # if [ -n "$xsde_rtsrc_unix_arch" ]; then for bk in "${!bins[@]}"; do pkg="$(sed -rn -e 's#^.+/.+/.+/(.+)/.+$#\1#p' <<<"$bk")" win="$(sed -rn -e 's#^.+/(windows.*)/.+/.+/.+$#true#p' <<<"$bk")" if [ "$pkg" == "xsde" -a -z "$win" ]; then bv="${bins[$bk]}" ln -s "$(sed -r -e 's/[^/]+/../g' <<<"$bv")/$xsde_rtsrc_unix_arch" \ "$bv/$xsde_rtsrc_unix_arch" fi done fi fi # Generate packages.sha256 # for d in "${!dirs[@]}"; do cd "$d" rm -f packages.sha256 l="$(find -L . -maxdepth 1 -type f -printf '%P\n' | sort)" # Array-like. sha256sum -b $l >packages.sha256 cd "$owd" done