aboutsummaryrefslogtreecommitdiff
path: root/server/mrrepo
blob: e22e5dd864effe44c456db6d63f7f254589f4202 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
#! /usr/bin/env bash

# Mirror repositories. For example:
#
# mrrepo -s git.example.org /var/scm
#
# Will first download (via http or https if -s specified) the manifest file
# from git.example.org which should list all publicly available repositories.
# It will then mirror each repository in /var/scm using the git protocol.
#
# -v
#  Run verbose.
#
# -s
#  Use https rather than http to download the manifest (git protocol is still
#  used for mirroring).
#
# Notes:
#   - needs curl
#   - run from cron as user scm (which belongs to the group scm).
#
usage="usage: $0 [-v] [-s] <host> <path>"

owd="$(pwd)"
trap "{ cd '$owd'; exit 1; }" ERR
set -o errtrace # Trap in functions.

function info () { echo "$*" 1>&2; }
function error () { info "$*"; exit 1; }

prot="http"
host=
path=
verb=0

while [ "$#" -gt 0 ]; do
  case "$1" in
    -v)
      verb=1
      shift
      ;;
    -s)
      prot="https"
      shift
      ;;
    *)
      if [ -z "$host" ]; then
        host="$1"
      elif [ -z "$path" ]; then
        path="${1%/}"
      else
        error "$usage"
      fi
      shift
      ;;
  esac
done

if [ -z "$host" -o -z "$path" ]; then
  error "$usage"
fi

if [ ! -d "$path" ]; then
  error "$path is not a directory"
fi

cd "$path"

curl_ops=()
curl_ops+=(-f)            # Fail on HTTP errors.
curl_ops+=(--max-time 30) # Finish in 30 seconds.

if [ "$verb" -ge 1 ]; then
  curl_ops+=(--progress-bar)
else
  curl_ops+=(-s -S)       # Silent but show errors.
fi

function fetch () # <url> [<curl-options>]
{
  local u="$1"; shift

  if [ "$verb" -ge 1 ]; then
    info "${curl_ops[@]}" "$@" "$u"
  fi

  curl "${curl_ops[@]}" "$@" "$u"
}

fetch "$prot://$host/manifest" -z manifest -o manifest

new=()
while read r || [ -n "$r" ]; do
  new+=("$r")
done <manifest

# Find all the existing repositories (directories that end with .git).
#
old=($(find . -type d -name '*.git' -print -prune | sed -e 's%^./%%' -))

git_ops=()
if [ "$verb" -eq 0 ]; then
  git_ops+=(-q)
fi

for r in "${new[@]}"; do
  if [ -d "$r" ]; then
    if [ -z "$(ls -A "$r")" ]; then
      rm -r "$r"
    fi
  fi
  if [ ! -d "$r" ]; then
    if [ "$verb" -ge 1 ]; then
      info "new repository $r in manifest, cloning"
      info git clone "${git_ops[@]}" --mirror "git://$host/$r" "$r"
    fi
    mkdir -p "$r"
    git clone "${git_ops[@]}" --mirror "git://$host/$r" "$r"

    # Also copy the description file.
    #
    fetch "$prot://$host/$r/description" -o "$r/description"
  else
    if [ "$verb" -ge 1 ]; then
      info "existing repository $r, fetching"
      info git -C "$r" fetch "${git_ops[@]}" --prune --tags
    fi
    git -C "$r" fetch "${git_ops[@]}" --prune --tags
  fi
done

# Remove old repositories.
#
for o in "${old[@]}"; do
  for n in "${new[@]}"; do
    if [ "$o" = "$n" ]; then
      o=
      break
    fi
  done

  if [ -n "$o" ]; then
    if [ "$verb" -ge 1 ]; then
      info "repository $o is no longer in manifest, removing"
    fi
    rm -rf "$o"
  fi
done

cd "$owd"