path: root/server/mrrepo

#! /usr/bin/env bash

# Mirror repositories. For example:
#
# mrrepo -s git.example.org /var/scm
#
# Will first download (via http or https if -s specified) the manifest file
# from git.example.org which should list all publicly available repositories.
# It will then mirror each remote repository in /var/scm locally, using the
# git protocol.
#
# Afterwards it may mirror it further to a remote repository that can be
# specified in the manifest file. If mirroring via https, then you also most
# likely need to provide credentials for the remote https URLs in the
# mrrepo-config file. This file should be placed next to and will be sourced
# by the mrrepo script (remember to adjust its permissions).
#
# The manifest file line format (lines starting with # are ignored):
#
# <repository-path>[ <remote-mirror-url>]
#
# To specify another credential for a URL add the following line to
# mrrepo-config:
#
# credentials['<https-url-prefix>']='<user>:<password>'
#
# -v
#  Run verbose.
#
# -s
#  Use https rather than http to download the manifest (git protocol is still
#  used for mirroring).
#
# Notes:
#   - needs curl
#   - run from cron as user scm (which belongs to the group scm).
#
# To test, run:
#
# runuser -u scm -- /var/scm/mrrepo -s -v git.example.org /var/scm
#
usage="usage: $0 [-v] [-s] <host> <path>"

owd="$(pwd)"
trap "{ cd '$owd'; exit 1; }" ERR
set -o errtrace # Trap in functions.

function info () { echo "$*" 1>&2; }
function error () { info "$*"; exit 1; }

prot="http"
host=
path=
verb=0

while [ "$#" -gt 0 ]; do
  case "$1" in
    -v)
      verb=1
      shift
      ;;
    -s)
      prot="https"
      shift
      ;;
    *)
      if [ -z "$host" ]; then
        host="$1"
      elif [ -z "$path" ]; then
        path="${1%/}"
      else
        error "$usage"
      fi
      shift
      ;;
  esac
done

if [ -z "$host" -o -z "$path" ]; then
  error "$usage"
fi

if [ ! -d "$path" ]; then
  error "$path is not a directory"
fi

declare -A credentials
config="$(realpath "${BASH_SOURCE[0]}")-config"

if [ -f "$config" ]; then
  source "$config"

  for p in "${!credentials[@]}"; do
    if [ "${p:0:8}" != "https://" ]; then
      error "https protocol is expected for '$p' in '$config'"
    fi
  done
fi

cd "$path"

curl_ops=()
curl_ops+=(-f)            # Fail on HTTP errors.
curl_ops+=(--max-time 30) # Finish in 30 seconds.

if [ "$verb" -ge 1 ]; then
  curl_ops+=(--progress-bar)
else
  curl_ops+=(-s -S)       # Silent but show errors.
fi

function fetch () # <url> [<curl-options>]
{
  local u="$1"; shift

  if [ "$verb" -ge 1 ]; then
    info curl "${curl_ops[@]}" "$@" "$u"
  fi

  curl "${curl_ops[@]}" "$@" "$u"
}

fetch "$prot://$host/manifest" -z manifest -o manifest

function field () # <line> <num> [<name>]
{
  local r
  r="$(echo "$1 " | cut -d ' ' -f "$2")"

  if [ "$3" -a -z "$r" ]; then
    error "field <$3> (#$2) missing in '$1'"
  fi
  echo "$r"
}

# Collect new repositories (in the new array) and while at it fix up remote
# URLs with credentials (in the auth_remotes map). Note that we still save
# original remote URLs to use them for diagnostics not to expose credentials
# (think about cron job diagnostics sent by email).
#
new=()
declare -A orig_remotes
declare -A auth_remotes

while read l || [ -n "$l" ]; do
  r=$(field "$l" 1 'path')
  u=$(field "$l" 2)

  new+=("$r")

  # If the remote URL is specified then add credentials into it, if found.
  # Note that currently we only support adding credentials for https URLs.
  #
  if [ -n "$u" ]; then
    orig_remotes["$r"]="$u"

    for p in "${!credentials[@]}"; do
      if [[ "$u" == "$p"* ]]; then
        c="${credentials[$p]}"
        u="$(echo "$u" | sed 's%^\(https://\)\(.*\)$%\1'"$c"'@\2%')"
	break;
      fi
    done

    auth_remotes["$r"]="$u"
  fi
done < <(sed -e '/^\s*#/d;/^\s*$/d;s/\s\s*/ /g' manifest)

# Find all the existing repositories (directories that end with .git).
#
old=($(find . -type d -name '*.git' -print -prune | sed -e 's%^./%%' -))

git_ops=()
if [ "$verb" -eq 0 ]; then
  git_ops+=(-q)
fi

for r in "${new[@]}"; do
  if [ -d "$r" ]; then
    if [ -z "$(ls -A "$r")" ]; then
      rm -r "$r"
    fi
  fi
  if [ ! -d "$r" ]; then
    if [ "$verb" -ge 1 ]; then
      info "new repository $r in manifest, cloning"
      info git clone "${git_ops[@]}" --mirror "git://$host/$r" "$r"
    fi
    mkdir -p "$r"
    git clone "${git_ops[@]}" --mirror "git://$host/$r" "$r"

    # Also copy the description file.
    #
    fetch "$prot://$host/$r/description" -o "$r/description"
  else
    if [ "$verb" -ge 1 ]; then
      info "existing repository $r, fetching"
      info git -C "$r" fetch "${git_ops[@]}" --prune --tags
    fi
    git -C "$r" fetch "${git_ops[@]}" --prune --tags

    # Also update the description file.
    #
    fetch "$prot://$host/$r/description" -z "$r/description" -o "$r/description"
  fi

  # Mirror to the remote URL, if present.
  #
  au="${auth_remotes[$r]}"
  if [ -n "$au" ]; then
    cmd=( git -C "$r" push "${git_ops[@]}" --mirror "$au" )

    # Note that in the verbose mode, for troubleshooting, we still print the
    # URLs that possibly contain credentials.
    #
    if [ "$verb" -ge 1 ]; then
      info "remote URL $au for repository $r, pushing"
      info "${cmd[@]}"
    fi

    # Disable prompting for username/password if credentials are missing for
    # the remote URL and fail instead.
    #
    # If the remote URL differs from the original one then it contains
    # credentials. It may potentially appear in git's STDERR, so we replace all
    # its occurrences with the original one, not containing credentials.
    #
    ou="${orig_remotes[$r]}"
    if [ "$au" != "$ou" ]; then
      GIT_TERMINAL_PROMPT=0 "${cmd[@]}" 2>&1 | sed "s%$au%$ou%g" >&2
    else
      GIT_TERMINAL_PROMPT=0 "${cmd[@]}"
    fi
  fi
done

# Remove old repositories.
#
for o in "${old[@]}"; do
  for n in "${new[@]}"; do
    if [ "$o" = "$n" ]; then
      o=
      break
    fi
  done

  if [ -n "$o" ]; then
    if [ "$verb" -ge 1 ]; then
      info "repository $o is no longer in manifest, removing"
    fi
    rm -rf "$o"
  fi
done

cd "$owd"