#! /usr/bin/env bash # Mirror repositories. For example: # # mrrepo -s git.example.org /var/scm # # Will first download (via http or https if -s specified) the manifest file # from git.example.org which should list all publicly available repositories. # It will then mirror each remote repository in /var/scm locally, using the # git protocol. # # Afterwards it may mirror it further to a remote repository that can be # specified in the manifest file. If mirroring via https, then you also most # likely need to provide credentials for the remote https URLs in the # mrrepo-config file. This file should be placed next to and will be sourced # by the mrrepo script (remember to adjust its permissions). # # The manifest file line format (lines starting with # are ignored): # # [ ] # # To specify another credential for a URL add the following line to # mrrepo-config: # # credentials['']=':' # # -v # Run verbose. # # -s # Use https rather than http to download the manifest (git protocol is still # used for mirroring). # # Notes: # - needs curl # - run from cron as user scm (which belongs to the group scm). # # To test, run: # # runuser -u scm -- /var/scm/mrrepo -s -v git.example.org /var/scm # usage="usage: $0 [-v] [-s] " owd="$(pwd)" trap "{ cd '$owd'; exit 1; }" ERR set -o errtrace # Trap in functions. function info () { echo "$*" 1>&2; } function error () { info "$*"; exit 1; } prot="http" host= path= verb=0 while [ "$#" -gt 0 ]; do case "$1" in -v) verb=1 shift ;; -s) prot="https" shift ;; *) if [ -z "$host" ]; then host="$1" elif [ -z "$path" ]; then path="${1%/}" else error "$usage" fi shift ;; esac done if [ -z "$host" -o -z "$path" ]; then error "$usage" fi if [ ! -d "$path" ]; then error "$path is not a directory" fi declare -A credentials config="$(realpath "${BASH_SOURCE[0]}")-config" if [ -f "$config" ]; then source "$config" for p in "${!credentials[@]}"; do if [ "${p:0:8}" != "https://" ]; then error "https protocol is expected for '$p' in '$config'" fi done fi cd "$path" curl_ops=() curl_ops+=(-f) # Fail on HTTP errors. curl_ops+=(--max-time 30) # Finish in 30 seconds. if [ "$verb" -ge 1 ]; then curl_ops+=(--progress-bar) else curl_ops+=(-s -S) # Silent but show errors. fi function fetch () # [] { local u="$1"; shift if [ "$verb" -ge 1 ]; then info curl "${curl_ops[@]}" "$@" "$u" fi curl "${curl_ops[@]}" "$@" "$u" } fetch "$prot://$host/manifest" -z manifest -o manifest function field () # [] { local r r="$(echo "$1 " | cut -d ' ' -f "$2")" if [ "$3" -a -z "$r" ]; then error "field <$3> (#$2) missing in '$1'" fi echo "$r" } # Collect new repositories (in the new array) and while at it fix up remote # URLs with credentials (in the remotes map). # new=() declare -A remotes while read l || [ -n "$l" ]; do r=$(field "$l" 1 'path') u=$(field "$l" 2) new+=("$r") # If the remote URL is specified then add credentials into it, if found. # Note that currently we only support adding credentials for https URLs. # if [ -n "$u" ]; then for p in "${!credentials[@]}"; do if [[ "$u" == "$p"* ]]; then c="${credentials[$p]}" u="$(echo "$u" | sed 's%^\(https://\)\(.*\)$%\1'"$c"'@\2%')" break; fi done remotes["$r"]="$u" fi done < <(sed -e '/^\s*#/d;/^\s*$/d;s/\s\s*/ /g' manifest) # Find all the existing repositories (directories that end with .git). # old=($(find . -type d -name '*.git' -print -prune | sed -e 's%^./%%' -)) git_ops=() if [ "$verb" -eq 0 ]; then git_ops+=(-q) fi for r in "${new[@]}"; do if [ -d "$r" ]; then if [ -z "$(ls -A "$r")" ]; then rm -r "$r" fi fi if [ ! -d "$r" ]; then if [ "$verb" -ge 1 ]; then info "new repository $r in manifest, cloning" info git clone "${git_ops[@]}" --mirror "git://$host/$r" "$r" fi mkdir -p "$r" git clone "${git_ops[@]}" --mirror "git://$host/$r" "$r" # Also copy the description file. # fetch "$prot://$host/$r/description" -o "$r/description" else if [ "$verb" -ge 1 ]; then info "existing repository $r, fetching" info git -C "$r" fetch "${git_ops[@]}" --prune --tags fi git -C "$r" fetch "${git_ops[@]}" --prune --tags # Also update the description file. # fetch "$prot://$host/$r/description" -z "$r/description" -o "$r/description" fi # Mirror to the remote URL, if present. # u="${remotes[$r]}" if [ -n "$u" ]; then if [ "$verb" -ge 1 ]; then info "remote URL $u for repository $r, pushing" info git -C "$r" push "${git_ops[@]}" --mirror "$u" fi # Disable prompting for username/password if credentials are missing for # the remote URL and fail instead. # GIT_TERMINAL_PROMPT=0 git -C "$r" push "${git_ops[@]}" --mirror "$u" fi done # Remove old repositories. # for o in "${old[@]}"; do for n in "${new[@]}"; do if [ "$o" = "$n" ]; then o= break fi done if [ -n "$o" ]; then if [ "$verb" -ge 1 ]; then info "repository $o is no longer in manifest, removing" fi rm -rf "$o" fi done cd "$owd"