aboutsummaryrefslogtreecommitdiff
path: root/server/mrrepo
diff options
context:
space:
mode:
Diffstat (limited to 'server/mrrepo')
-rwxr-xr-xserver/mrrepo220
1 files changed, 163 insertions, 57 deletions
diff --git a/server/mrrepo b/server/mrrepo
index 029ce35..1bb3989 100755
--- a/server/mrrepo
+++ b/server/mrrepo
@@ -6,14 +6,15 @@
#
# Will first download (via http or https if -s specified) the manifest file
# from git.example.org which should list all publicly available repositories.
-# It will then mirror each remote repository in /var/scm locally, using the
-# git protocol.
+# It will then pull-mirror each remote repository locally in /var/scm using
+# the git protocol.
#
-# Afterwards it may mirror it further to a remote repository that can be
-# specified in the manifest file. If mirroring via https, then you also most
-# likely need to provide credentials for the remote https URLs in the
-# mrrepo-config file. This file should be placed next to and will be sourced
-# by the mrrepo script (remember to adjust its permissions).
+# Afterwards it may push-mirror them as well as local repositories (specified
+# in the local manifest) further to a remote repository that can be specified
+# in the manifest files. If mirroring via https, then you also most likely
+# need to provide credentials for the remote https URLs in the mrrepo-config
+# file. This file should be placed next to and will be sourced by the mrrepo
+# script (remember to adjust its permissions).
#
# The manifest file line format (lines starting with # are ignored):
#
@@ -120,9 +121,14 @@ function fetch () # <url> [<curl-options>]
curl "${curl_ops[@]}" "$@" "$u"
}
-fetch "$prot://$host/manifest" -z manifest -o manifest
+fetch "$prot://$host/manifest" -z remote.manifest -o remote.manifest
-function field () # <line> <num> [<name>]
+function manifest_filter () # <file>
+{
+ sed -e '/^\s*#/d;/^\s*$/d;s/\s\s*/ /g' "$1"
+}
+
+function manifest_field () # <line> <num> [<name>]
{
local r
r="$(echo "$1 " | cut -d ' ' -f "$2")"
@@ -133,66 +139,141 @@ function field () # <line> <num> [<name>]
echo "$r"
}
-# Collect new repositories (in the new array) and while at it fix up remote
-# URLs with credentials (in the auth_remotes map). Note that we still save
-# original remote URLs to use them for diagnostics not to expose credentials
-# (think about cron job diagnostics sent by email).
+# Collect remote repositories (in the remote array) and while at it fix up
+# push URLs with credentials (in the push_auth map). Note that we also save
+# the original push URLs (in push_orig) to use them for diagnostics so that we
+# don't expose credentials (think about cron job diagnostics sent by email).
#
-new=()
-declare -A orig_remotes
-declare -A auth_remotes
+remote=()
+declare -A push_orig
+declare -A push_auth
+
+function push_add () # <rep> <url>
+{
+ # Note that currently we only support adding credentials for https URLs.
+ #
+ local r="$1"
+ local u="$2"
+
+ push_orig["$r"]="$u"
+
+ local p c
+ for p in "${!credentials[@]}"; do
+ if [[ "$u" == "$p"* ]]; then
+ c="${credentials[$p]}"
+ u="$(echo "$u" | sed 's%^\(https://\)\(.*\)$%\1'"$c"'@\2%')"
+ break;
+ fi
+ done
+
+ push_auth["$r"]="$u"
+}
while read l || [ -n "$l" ]; do
- r=$(field "$l" 1 'path')
- u=$(field "$l" 2)
+ r=$(manifest_field "$l" 1 'path')
+ u=$(manifest_field "$l" 2)
- new+=("$r")
+ remote+=("$r")
- # If the remote URL is specified then add credentials into it, if found.
- # Note that currently we only support adding credentials for https URLs.
+ # If the push URL is specified then add it to auth/orig maps.
#
if [ -n "$u" ]; then
- orig_remotes["$r"]="$u"
+ push_add "$r" "$u"
+ fi
+done < <(manifest_filter remote.manifest)
+
+# Find all the existing repositories (directories that end with .git) and sort
+# them out into mirrored and local public. Note that local private will end up
+# in the mirrored array and will require ad hoc handling.
+#
+all=($(find . -type d -name '*.git' -print -prune | sed -e 's%^./%%' -))
+
+mirror=()
+local=()
+
+# If we have local manifest, load its repositories and also verify they are
+# not in remotes. Also add their push URLs similar to remotes.
+#
+if test -f manifest; then
- for p in "${!credentials[@]}"; do
- if [[ "$u" == "$p"* ]]; then
- c="${credentials[$p]}"
- u="$(echo "$u" | sed 's%^\(https://\)\(.*\)$%\1'"$c"'@\2%')"
- break;
+ while read l || [ -n "$l" ]; do
+ r=$(manifest_field "$l" 1 'path')
+ u=$(manifest_field "$l" 2)
+
+ for i in "${remote[@]}"; do
+ if [ "$i" = "$r" ]; then
+ error "attempt to mirror into local public repository $r"
fi
done
- auth_remotes["$r"]="$u"
- fi
-done < <(sed -e '/^\s*#/d;/^\s*$/d;s/\s\s*/ /g' manifest)
+ local+=("$r")
-# Find all the existing repositories (directories that end with .git).
-#
-old=($(find . -type d -name '*.git' -print -prune | sed -e 's%^./%%' -))
+ # If the push URL is specified then add it to auth/orig maps.
+ #
+ if [ -n "$u" ]; then
+ push_add "$r" "$u"
+ fi
+ done < <(manifest_filter manifest)
+
+ # Everything that is not in local is mirrored (or local private).
+ #
+ for r in "${all[@]}"; do
+
+ for i in "${local[@]}"; do
+ if [ "$i" = "$r" ]; then
+ if [ "$verb" -ge 1 ]; then
+ info "local public repository $r"
+ fi
+ r=
+ break
+ fi
+ done
+
+ if [ -n "$r" ]; then
+ mirror+=("$r")
+ fi
+ done
+else
+ mirror=("${all[@]}")
+fi
git_ops=()
if [ "$verb" -eq 0 ]; then
git_ops+=(-q)
fi
-for r in "${new[@]}"; do
+for r in "${remote[@]}"; do
+
+ # Zap empty directories.
+ #
if [ -d "$r" ]; then
if [ -z "$(ls -A "$r")" ]; then
rm -r "$r"
fi
fi
+
if [ ! -d "$r" ]; then
+
if [ "$verb" -ge 1 ]; then
- info "new repository $r in manifest, cloning"
+ info "new repository $r in remote manifest, cloning"
info git clone "${git_ops[@]}" --mirror "git://$host/$r" "$r"
fi
+
mkdir -p "$r"
git clone "${git_ops[@]}" --mirror "git://$host/$r" "$r"
# Also copy the description file.
#
fetch "$prot://$host/$r/description" -o "$r/description"
+
else
+
+ # Make sure it is not a local private repository.
+ #
+ if test ! -f "$r/git-daemon-export-ok"; then
+ error "attempt to mirror into local private repository $r"
+ fi
+
if [ "$verb" -ge 1 ]; then
info "existing repository $r, fetching"
info git -C "$r" fetch "${git_ops[@]}" --prune --tags
@@ -204,9 +285,46 @@ for r in "${new[@]}"; do
fetch "$prot://$host/$r/description" -z "$r/description" -o "$r/description"
fi
- # Mirror to the remote URL, if present.
+ # Mark as public.
#
- au="${auth_remotes[$r]}"
+ if test ! -f "$r/git-daemon-export-ok"; then
+ touch "$r/git-daemon-export-ok"
+ fi
+done
+
+# Remove old mirrored repositories.
+#
+for o in "${mirror[@]}"; do
+
+ # Don't touch if it's local private repository.
+ #
+ if test ! -f "$o/git-daemon-export-ok"; then
+ if [ "$verb" -ge 1 ]; then
+ info "skipping local private repository $o"
+ fi
+ continue
+ fi
+
+ for i in "${remote[@]}"; do
+ if [ "$i" = "$o" ]; then
+ o=
+ break
+ fi
+ done
+
+ if [ -n "$o" ]; then
+ if [ "$verb" -ge 1 ]; then
+ info "repository $o is no longer in remote manifest, removing"
+ fi
+ rm -rf "$o"
+ fi
+done
+
+# Mirror to the push URLs.
+#
+for r in "${!push_auth[@]}"; do
+
+ au="${push_auth[$r]}"
if [ -n "$au" ]; then
cmd=( git -C "$r" push "${git_ops[@]}" --mirror "$au" )
@@ -225,30 +343,18 @@ for r in "${new[@]}"; do
# credentials. It may potentially appear in git's STDERR, so we replace all
# its occurrences with the original one, not containing credentials.
#
- ou="${orig_remotes[$r]}"
+ ou="${push_orig[$r]}"
if [ "$au" != "$ou" ]; then
- GIT_TERMINAL_PROMPT=0 "${cmd[@]}" 2>&1 | sed "s%$au%$ou%g" >&2
- else
- GIT_TERMINAL_PROMPT=0 "${cmd[@]}"
- fi
- fi
-done
-# Remove old repositories.
-#
-for o in "${old[@]}"; do
- for n in "${new[@]}"; do
- if [ "$o" = "$n" ]; then
- o=
- break
- fi
- done
+ # Escape special characters in sed pattern/substitution.
+ #
+ au="$(sed -e 's/[].*/\[]/\\&/g' <<<"$au")"
+ ou="$(sed -e 's/[&/\]/\\&/g' <<<"$ou")"
- if [ -n "$o" ]; then
- if [ "$verb" -ge 1 ]; then
- info "repository $o is no longer in manifest, removing"
+ GIT_TERMINAL_PROMPT=0 "${cmd[@]}" 2>&1 | sed "s/$au/$ou/g" >&2
+ else
+ GIT_TERMINAL_PROMPT=0 "${cmd[@]}"
fi
- rm -rf "$o"
fi
done