Update mrrepo script to support local repositories

1 files changed, 163 insertions, 57 deletions

diff --git a/server/mrrepo b/server/mrrepo
index 029ce35..1bb3989 100755
--- a/server/mrrepo
+++ b/server/mrrepo
@@ -6,14 +6,15 @@
 #
 # Will first download (via http or https if -s specified) the manifest file
 # from git.example.org which should list all publicly available repositories.
-# It will then mirror each remote repository in /var/scm locally, using the
-# git protocol.
+# It will then pull-mirror each remote repository locally in /var/scm using
+# the git protocol.
 #
-# Afterwards it may mirror it further to a remote repository that can be
-# specified in the manifest file. If mirroring via https, then you also most
-# likely need to provide credentials for the remote https URLs in the
-# mrrepo-config file. This file should be placed next to and will be sourced
-# by the mrrepo script (remember to adjust its permissions).
+# Afterwards it may push-mirror them as well as local repositories (specified
+# in the local manifest) further to a remote repository that can be specified
+# in the manifest files. If mirroring via https, then you also most likely
+# need to provide credentials for the remote https URLs in the mrrepo-config
+# file. This file should be placed next to and will be sourced by the mrrepo
+# script (remember to adjust its permissions).
 #
 # The manifest file line format (lines starting with # are ignored):
 #
@@ -120,9 +121,14 @@ function fetch () # <url> [<curl-options>]
   curl "${curl_ops[@]}" "$@" "$u"
 }
 
-fetch "$prot://$host/manifest" -z manifest -o manifest
+fetch "$prot://$host/manifest" -z remote.manifest -o remote.manifest
 
-function field () # <line> <num> [<name>]
+function manifest_filter () # <file>
+{
+  sed -e '/^\s*#/d;/^\s*$/d;s/\s\s*/ /g' "$1"
+}
+
+function manifest_field () # <line> <num> [<name>]
 {
   local r
   r="$(echo "$1 " | cut -d ' ' -f "$2")"
@@ -133,66 +139,141 @@ function field () # <line> <num> [<name>]
   echo "$r"
 }
 
-# Collect new repositories (in the new array) and while at it fix up remote
-# URLs with credentials (in the auth_remotes map). Note that we still save
-# original remote URLs to use them for diagnostics not to expose credentials
-# (think about cron job diagnostics sent by email).
+# Collect remote repositories (in the remote array) and while at it fix up
+# push URLs with credentials (in the push_auth map). Note that we also save
+# the original push URLs (in push_orig) to use them for diagnostics so that we
+# don't expose credentials (think about cron job diagnostics sent by email).
 #
-new=()
-declare -A orig_remotes
-declare -A auth_remotes
+remote=()
+declare -A push_orig
+declare -A push_auth
+
+function push_add () # <rep> <url>
+{
+  # Note that currently we only support adding credentials for https URLs.
+  #
+  local r="$1"
+  local u="$2"
+
+  push_orig["$r"]="$u"
+
+  local p c
+  for p in "${!credentials[@]}"; do
+    if [[ "$u" == "$p"* ]]; then
+      c="${credentials[$p]}"
+      u="$(echo "$u" | sed 's%^\(https://\)\(.*\)$%\1'"$c"'@\2%')"
+      break;
+    fi
+  done
+
+  push_auth["$r"]="$u"
+}
 
 while read l || [ -n "$l" ]; do
-  r=$(field "$l" 1 'path')
-  u=$(field "$l" 2)
+  r=$(manifest_field "$l" 1 'path')
+  u=$(manifest_field "$l" 2)
 
-  new+=("$r")
+  remote+=("$r")
 
-  # If the remote URL is specified then add credentials into it, if found.
-  # Note that currently we only support adding credentials for https URLs.
+  # If the push URL is specified then add it to auth/orig maps.
   #
   if [ -n "$u" ]; then
-    orig_remotes["$r"]="$u"
+    push_add "$r" "$u"
+  fi
+done < <(manifest_filter remote.manifest)
+
+# Find all the existing repositories (directories that end with .git) and sort
+# them out into mirrored and local public. Note that local private will end up
+# in the mirrored array and will require ad hoc handling.
+#
+all=($(find . -type d -name '*.git' -print -prune | sed -e 's%^./%%' -))
+
+mirror=()
+local=()
+
+# If we have local manifest, load its repositories and also verify they are
+# not in remotes. Also add their push URLs similar to remotes.
+#
+if test -f manifest; then
 
-    for p in "${!credentials[@]}"; do
-      if [[ "$u" == "$p"* ]]; then
-        c="${credentials[$p]}"
-        u="$(echo "$u" | sed 's%^\(https://\)\(.*\)$%\1'"$c"'@\2%')"
-	break;
+  while read l || [ -n "$l" ]; do
+    r=$(manifest_field "$l" 1 'path')
+    u=$(manifest_field "$l" 2)
+
+    for i in "${remote[@]}"; do
+      if [ "$i" = "$r" ]; then
+	error "attempt to mirror into local public repository $r"
       fi
     done
 
-    auth_remotes["$r"]="$u"
-  fi
-done < <(sed -e '/^\s*#/d;/^\s*$/d;s/\s\s*/ /g' manifest)
+    local+=("$r")
 
-# Find all the existing repositories (directories that end with .git).
-#
-old=($(find . -type d -name '*.git' -print -prune | sed -e 's%^./%%' -))
+    # If the push URL is specified then add it to auth/orig maps.
+    #
+    if [ -n "$u" ]; then
+      push_add "$r" "$u"
+    fi
+  done < <(manifest_filter manifest)
+
+  # Everything that is not in local is mirrored (or local private).
+  #
+  for r in "${all[@]}"; do
+
+    for i in "${local[@]}"; do
+      if [ "$i" = "$r" ]; then
+        if [ "$verb" -ge 1 ]; then
+	  info "local public repository $r"
+	fi
+	r=
+	break
+      fi
+    done
+
+    if [ -n "$r" ]; then
+      mirror+=("$r")
+    fi
+  done
+else
+  mirror=("${all[@]}")
+fi
 
 git_ops=()
 if [ "$verb" -eq 0 ]; then
   git_ops+=(-q)
 fi
 
-for r in "${new[@]}"; do
+for r in "${remote[@]}"; do
+
+  # Zap empty directories.
+  #
   if [ -d "$r" ]; then
     if [ -z "$(ls -A "$r")" ]; then
       rm -r "$r"
     fi
   fi
+
   if [ ! -d "$r" ]; then
+
     if [ "$verb" -ge 1 ]; then
-      info "new repository $r in manifest, cloning"
+      info "new repository $r in remote manifest, cloning"
       info git clone "${git_ops[@]}" --mirror "git://$host/$r" "$r"
     fi
+
     mkdir -p "$r"
     git clone "${git_ops[@]}" --mirror "git://$host/$r" "$r"
 
     # Also copy the description file.
     #
     fetch "$prot://$host/$r/description" -o "$r/description"
+
   else
+
+    # Make sure it is not a local private repository.
+    #
+    if test ! -f "$r/git-daemon-export-ok"; then
+      error "attempt to mirror into local private repository $r"
+    fi
+
     if [ "$verb" -ge 1 ]; then
       info "existing repository $r, fetching"
       info git -C "$r" fetch "${git_ops[@]}" --prune --tags
@@ -204,9 +285,46 @@ for r in "${new[@]}"; do
     fetch "$prot://$host/$r/description" -z "$r/description" -o "$r/description"
   fi
 
-  # Mirror to the remote URL, if present.
+  # Mark as public.
   #
-  au="${auth_remotes[$r]}"
+  if test ! -f "$r/git-daemon-export-ok"; then
+     touch "$r/git-daemon-export-ok"
+  fi
+done
+
+# Remove old mirrored repositories.
+#
+for o in "${mirror[@]}"; do
+
+  # Don't touch if it's local private repository.
+  #
+  if test ! -f "$o/git-daemon-export-ok"; then
+    if [ "$verb" -ge 1 ]; then
+      info "skipping local private repository $o"
+    fi
+    continue
+  fi
+
+  for i in "${remote[@]}"; do
+    if [ "$i" = "$o" ]; then
+      o=
+      break
+    fi
+  done
+
+  if [ -n "$o" ]; then
+    if [ "$verb" -ge 1 ]; then
+      info "repository $o is no longer in remote manifest, removing"
+    fi
+    rm -rf "$o"
+  fi
+done
+
+# Mirror to the push URLs.
+#
+for r in "${!push_auth[@]}"; do
+
+  au="${push_auth[$r]}"
   if [ -n "$au" ]; then
     cmd=( git -C "$r" push "${git_ops[@]}" --mirror "$au" )
 
@@ -225,30 +343,18 @@ for r in "${new[@]}"; do
     # credentials. It may potentially appear in git's STDERR, so we replace all
     # its occurrences with the original one, not containing credentials.
     #
-    ou="${orig_remotes[$r]}"
+    ou="${push_orig[$r]}"
     if [ "$au" != "$ou" ]; then
-      GIT_TERMINAL_PROMPT=0 "${cmd[@]}" 2>&1 | sed "s%$au%$ou%g" >&2
-    else
-      GIT_TERMINAL_PROMPT=0 "${cmd[@]}"
-    fi
-  fi
-done
 
-# Remove old repositories.
-#
-for o in "${old[@]}"; do
-  for n in "${new[@]}"; do
-    if [ "$o" = "$n" ]; then
-      o=
-      break
-    fi
-  done
+      # Escape special characters in sed pattern/substitution.
+      #
+      au="$(sed -e 's/[].*/\[]/\\&/g' <<<"$au")"
+      ou="$(sed -e 's/[&/\]/\\&/g'    <<<"$ou")"
 
-  if [ -n "$o" ]; then
-    if [ "$verb" -ge 1 ]; then
-      info "repository $o is no longer in manifest, removing"
+      GIT_TERMINAL_PROMPT=0 "${cmd[@]}" 2>&1 | sed "s/$au/$ou/g" >&2
+    else
+      GIT_TERMINAL_PROMPT=0 "${cmd[@]}"
     fi
-    rm -rf "$o"
   fi
 done