aboutsummaryrefslogtreecommitdiff
path: root/server/mrrepo
blob: 1bb398986e94ff3c92ee0239a89df5798cbeb12d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
#! /usr/bin/env bash

# Mirror repositories. For example:
#
# mrrepo -s git.example.org /var/scm
#
# Will first download (via http or https if -s specified) the manifest file
# from git.example.org which should list all publicly available repositories.
# It will then pull-mirror each remote repository locally in /var/scm using
# the git protocol.
#
# Afterwards it may push-mirror them as well as local repositories (specified
# in the local manifest) further to a remote repository that can be specified
# in the manifest files. If mirroring via https, then you also most likely
# need to provide credentials for the remote https URLs in the mrrepo-config
# file. This file should be placed next to and will be sourced by the mrrepo
# script (remember to adjust its permissions).
#
# The manifest file line format (lines starting with # are ignored):
#
# <repository-path>[ <remote-mirror-url>]
#
# To specify another credential for a URL add the following line to
# mrrepo-config:
#
# credentials['<https-url-prefix>']='<user>:<password>'
#
# -v
#  Run verbose.
#
# -s
#  Use https rather than http to download the manifest (git protocol is still
#  used for mirroring).
#
# Notes:
#   - needs curl
#   - run from cron as user scm (which belongs to the group scm).
#
# To test, run:
#
# runuser -u scm -- /var/scm/mrrepo -s -v git.example.org /var/scm
#
usage="usage: $0 [-v] [-s] <host> <path>"

owd="$(pwd)"
trap "{ cd '$owd'; exit 1; }" ERR
set -o errtrace # Trap in functions.

function info () { echo "$*" 1>&2; }
function error () { info "$*"; exit 1; }

prot="http"
host=
path=
verb=0

while [ "$#" -gt 0 ]; do
  case "$1" in
    -v)
      verb=1
      shift
      ;;
    -s)
      prot="https"
      shift
      ;;
    *)
      if [ -z "$host" ]; then
        host="$1"
      elif [ -z "$path" ]; then
        path="${1%/}"
      else
        error "$usage"
      fi
      shift
      ;;
  esac
done

if [ -z "$host" -o -z "$path" ]; then
  error "$usage"
fi

if [ ! -d "$path" ]; then
  error "$path is not a directory"
fi

declare -A credentials
config="$(realpath "${BASH_SOURCE[0]}")-config"

if [ -f "$config" ]; then
  source "$config"

  for p in "${!credentials[@]}"; do
    if [ "${p:0:8}" != "https://" ]; then
      error "https protocol is expected for '$p' in '$config'"
    fi
  done
fi

cd "$path"

curl_ops=()
curl_ops+=(-f)            # Fail on HTTP errors.
curl_ops+=(--max-time 30) # Finish in 30 seconds.

if [ "$verb" -ge 1 ]; then
  curl_ops+=(--progress-bar)
else
  curl_ops+=(-s -S)       # Silent but show errors.
fi

function fetch () # <url> [<curl-options>]
{
  local u="$1"; shift

  if [ "$verb" -ge 1 ]; then
    info curl "${curl_ops[@]}" "$@" "$u"
  fi

  curl "${curl_ops[@]}" "$@" "$u"
}

fetch "$prot://$host/manifest" -z remote.manifest -o remote.manifest

function manifest_filter () # <file>
{
  sed -e '/^\s*#/d;/^\s*$/d;s/\s\s*/ /g' "$1"
}

function manifest_field () # <line> <num> [<name>]
{
  local r
  r="$(echo "$1 " | cut -d ' ' -f "$2")"

  if [ "$3" -a -z "$r" ]; then
    error "field <$3> (#$2) missing in '$1'"
  fi
  echo "$r"
}

# Collect remote repositories (in the remote array) and while at it fix up
# push URLs with credentials (in the push_auth map). Note that we also save
# the original push URLs (in push_orig) to use them for diagnostics so that we
# don't expose credentials (think about cron job diagnostics sent by email).
#
remote=()
declare -A push_orig
declare -A push_auth

function push_add () # <rep> <url>
{
  # Note that currently we only support adding credentials for https URLs.
  #
  local r="$1"
  local u="$2"

  push_orig["$r"]="$u"

  local p c
  for p in "${!credentials[@]}"; do
    if [[ "$u" == "$p"* ]]; then
      c="${credentials[$p]}"
      u="$(echo "$u" | sed 's%^\(https://\)\(.*\)$%\1'"$c"'@\2%')"
      break;
    fi
  done

  push_auth["$r"]="$u"
}

while read l || [ -n "$l" ]; do
  r=$(manifest_field "$l" 1 'path')
  u=$(manifest_field "$l" 2)

  remote+=("$r")

  # If the push URL is specified then add it to auth/orig maps.
  #
  if [ -n "$u" ]; then
    push_add "$r" "$u"
  fi
done < <(manifest_filter remote.manifest)

# Find all the existing repositories (directories that end with .git) and sort
# them out into mirrored and local public. Note that local private will end up
# in the mirrored array and will require ad hoc handling.
#
all=($(find . -type d -name '*.git' -print -prune | sed -e 's%^./%%' -))

mirror=()
local=()

# If we have local manifest, load its repositories and also verify they are
# not in remotes. Also add their push URLs similar to remotes.
#
if test -f manifest; then

  while read l || [ -n "$l" ]; do
    r=$(manifest_field "$l" 1 'path')
    u=$(manifest_field "$l" 2)

    for i in "${remote[@]}"; do
      if [ "$i" = "$r" ]; then
	error "attempt to mirror into local public repository $r"
      fi
    done

    local+=("$r")

    # If the push URL is specified then add it to auth/orig maps.
    #
    if [ -n "$u" ]; then
      push_add "$r" "$u"
    fi
  done < <(manifest_filter manifest)

  # Everything that is not in local is mirrored (or local private).
  #
  for r in "${all[@]}"; do

    for i in "${local[@]}"; do
      if [ "$i" = "$r" ]; then
        if [ "$verb" -ge 1 ]; then
	  info "local public repository $r"
	fi
	r=
	break
      fi
    done

    if [ -n "$r" ]; then
      mirror+=("$r")
    fi
  done
else
  mirror=("${all[@]}")
fi

git_ops=()
if [ "$verb" -eq 0 ]; then
  git_ops+=(-q)
fi

for r in "${remote[@]}"; do

  # Zap empty directories.
  #
  if [ -d "$r" ]; then
    if [ -z "$(ls -A "$r")" ]; then
      rm -r "$r"
    fi
  fi

  if [ ! -d "$r" ]; then

    if [ "$verb" -ge 1 ]; then
      info "new repository $r in remote manifest, cloning"
      info git clone "${git_ops[@]}" --mirror "git://$host/$r" "$r"
    fi

    mkdir -p "$r"
    git clone "${git_ops[@]}" --mirror "git://$host/$r" "$r"

    # Also copy the description file.
    #
    fetch "$prot://$host/$r/description" -o "$r/description"

  else

    # Make sure it is not a local private repository.
    #
    if test ! -f "$r/git-daemon-export-ok"; then
      error "attempt to mirror into local private repository $r"
    fi

    if [ "$verb" -ge 1 ]; then
      info "existing repository $r, fetching"
      info git -C "$r" fetch "${git_ops[@]}" --prune --tags
    fi
    git -C "$r" fetch "${git_ops[@]}" --prune --tags

    # Also update the description file.
    #
    fetch "$prot://$host/$r/description" -z "$r/description" -o "$r/description"
  fi

  # Mark as public.
  #
  if test ! -f "$r/git-daemon-export-ok"; then
     touch "$r/git-daemon-export-ok"
  fi
done

# Remove old mirrored repositories.
#
for o in "${mirror[@]}"; do

  # Don't touch if it's local private repository.
  #
  if test ! -f "$o/git-daemon-export-ok"; then
    if [ "$verb" -ge 1 ]; then
      info "skipping local private repository $o"
    fi
    continue
  fi

  for i in "${remote[@]}"; do
    if [ "$i" = "$o" ]; then
      o=
      break
    fi
  done

  if [ -n "$o" ]; then
    if [ "$verb" -ge 1 ]; then
      info "repository $o is no longer in remote manifest, removing"
    fi
    rm -rf "$o"
  fi
done

# Mirror to the push URLs.
#
for r in "${!push_auth[@]}"; do

  au="${push_auth[$r]}"
  if [ -n "$au" ]; then
    cmd=( git -C "$r" push "${git_ops[@]}" --mirror "$au" )

    # Note that in the verbose mode, for troubleshooting, we still print the
    # URLs that possibly contain credentials.
    #
    if [ "$verb" -ge 1 ]; then
      info "remote URL $au for repository $r, pushing"
      info "${cmd[@]}"
    fi

    # Disable prompting for username/password if credentials are missing for
    # the remote URL and fail instead.
    #
    # If the remote URL differs from the original one then it contains
    # credentials. It may potentially appear in git's STDERR, so we replace all
    # its occurrences with the original one, not containing credentials.
    #
    ou="${push_orig[$r]}"
    if [ "$au" != "$ou" ]; then

      # Escape special characters in sed pattern/substitution.
      #
      au="$(sed -e 's/[].*/\[]/\\&/g' <<<"$au")"
      ou="$(sed -e 's/[&/\]/\\&/g'    <<<"$ou")"

      GIT_TERMINAL_PROMPT=0 "${cmd[@]}" 2>&1 | sed "s/$au/$ou/g" >&2
    else
      GIT_TERMINAL_PROMPT=0 "${cmd[@]}"
    fi
  fi
done

cd "$owd"