Use ad hoc recipe for parsing code and documentation generating

The overall approach is to store pre-generated bootstrap options.?xx
and cli.{1,xhtml} and automatically update them in the development
build (config.cli.develop=true). See README.md in the root of the
repository for details.

6 files changed, 220 insertions, 70 deletions

diff --git a/cli/cli/.gitignore b/cli/cli/.gitignore
index 903d015..79562f1 100644
--- a/cli/cli/.gitignore
+++ b/cli/cli/.gitignore
@@ -1,5 +1,7 @@
-cli
-version.hxx
+/cli
+/bootstrap/cli/cli
+/version.hxx
+/options.?xx
 
 # Unit test executables and Testscript output directories (can be symlinks).
 #
diff --git a/cli/cli/options.cxx b/cli/cli/bootstrap/cli/options.cxx
index 0f0604a..56dd24f 100644
--- a/cli/cli/options.cxx
+++ b/cli/cli/bootstrap/cli/options.cxx
@@ -15,6 +15,7 @@
 #include <set>
 #include <string>
 #include <vector>
+#include <utility>
 #include <ostream>
 #include <sstream>
 #include <cstring>
@@ -196,6 +197,7 @@ namespace cli
       else
         ++i_;
 
+      ++start_position_;
       return r;
     }
     else
@@ -206,11 +208,20 @@ namespace cli
   skip ()
   {
     if (i_ < argc_)
+    {
       ++i_;
+      ++start_position_;
+    }
     else
       throw eos_reached ();
   }
 
+  std::size_t argv_scanner::
+  position ()
+  {
+    return start_position_;
+  }
+
   // argv_file_scanner
   //
   int argv_file_scanner::zero_argc_ = 0;
@@ -321,6 +332,7 @@ namespace cli
     {
       hold_[i_ == 0 ? ++i_ : --i_].swap (args_.front ().value);
       args_.pop_front ();
+      ++start_position_;
       return hold_[i_].c_str ();
     }
   }
@@ -334,7 +346,10 @@ namespace cli
     if (args_.empty ())
       return base::skip ();
     else
+    {
       args_.pop_front ();
+      ++start_position_;
+    }
   }
 
   const argv_file_scanner::option_info* argv_file_scanner::
@@ -347,6 +362,12 @@ namespace cli
     return 0;
   }
 
+  std::size_t argv_file_scanner::
+  position ()
+  {
+    return start_position_;
+  }
+
   void argv_file_scanner::
   load (const std::string& file)
   {
@@ -546,6 +567,17 @@ namespace cli
   };
 
   template <typename X>
+  struct parser<std::pair<X, std::size_t> >
+  {
+    static void
+    parse (std::pair<X, std::size_t>& x, bool& xs, scanner& s)
+    {
+      x.second = s.position ();
+      parser<X>::parse (x.first, xs, s);
+    }
+  };
+
+  template <typename X>
   struct parser<std::vector<X> >
   {
     static void
@@ -583,6 +615,7 @@ namespace cli
 
       if (s.more ())
       {
+        std::size_t pos (s.position ());
         std::string ov (s.next ());
         std::string::size_type p = ov.find ('=');
 
@@ -602,14 +635,14 @@ namespace cli
         if (!kstr.empty ())
         {
           av[1] = const_cast<char*> (kstr.c_str ());
-          argv_scanner s (0, ac, av);
+          argv_scanner s (0, ac, av, false, pos);
           parser<K>::parse (k, dummy, s);
         }
 
         if (!vstr.empty ())
         {
           av[1] = const_cast<char*> (vstr.c_str ());
-          argv_scanner s (0, ac, av);
+          argv_scanner s (0, ac, av, false, pos);
           parser<V>::parse (v, dummy, s);
         }
 
diff --git a/cli/cli/options.hxx b/cli/cli/bootstrap/cli/options.hxx
index b54d81f..35108fa 100644
--- a/cli/cli/options.hxx
+++ b/cli/cli/bootstrap/cli/options.hxx
@@ -236,6 +236,14 @@ namespace cli
   // for the two previous arguments up until a call to a third
   // peek() or next().
   //
+  // The position() function returns a monotonically-increasing
+  // number which, if stored, can later be used to determine the
+  // relative position of the argument returned by the following
+  // call to next(). Note that if multiple scanners are used to
+  // extract arguments from multiple sources, then the end
+  // position of the previous scanner should be used as the
+  // start position of the next.
+  //
   class scanner
   {
     public:
@@ -253,13 +261,24 @@ namespace cli
 
     virtual void
     skip () = 0;
+
+    virtual std::size_t
+    position () = 0;
   };
 
   class argv_scanner: public scanner
   {
     public:
-    argv_scanner (int& argc, char** argv, bool erase = false);
-    argv_scanner (int start, int& argc, char** argv, bool erase = false);
+    argv_scanner (int& argc,
+                  char** argv,
+                  bool erase = false,
+                  std::size_t start_position = 0);
+
+    argv_scanner (int start,
+                  int& argc,
+                  char** argv,
+                  bool erase = false,
+                  std::size_t start_position = 0);
 
     int
     end () const;
@@ -276,7 +295,11 @@ namespace cli
     virtual void
     skip ();
 
-    private:
+    virtual std::size_t
+    position ();
+
+    protected:
+    std::size_t start_position_;
     int i_;
     int& argc_;
     char** argv_;
@@ -289,16 +312,19 @@ namespace cli
     argv_file_scanner (int& argc,
                        char** argv,
                        const std::string& option,
-                       bool erase = false);
+                       bool erase = false,
+                       std::size_t start_position = 0);
 
     argv_file_scanner (int start,
                        int& argc,
                        char** argv,
                        const std::string& option,
-                       bool erase = false);
+                       bool erase = false,
+                       std::size_t start_position = 0);
 
     argv_file_scanner (const std::string& file,
-                       const std::string& option);
+                       const std::string& option,
+                       std::size_t start_position = 0);
 
     struct option_info
     {
@@ -315,18 +341,21 @@ namespace cli
                         char** argv,
                         const option_info* options,
                         std::size_t options_count,
-                        bool erase = false);
+                        bool erase = false,
+                        std::size_t start_position = 0);
 
     argv_file_scanner (int start,
                        int& argc,
                        char** argv,
                        const option_info* options,
                        std::size_t options_count,
-                       bool erase = false);
+                       bool erase = false,
+                       std::size_t start_position = 0);
 
     argv_file_scanner (const std::string& file,
                        const option_info* options = 0,
-                       std::size_t options_count = 0);
+                       std::size_t options_count = 0,
+                       std::size_t start_position = 0);
 
     virtual bool
     more ();
@@ -340,6 +369,9 @@ namespace cli
     virtual void
     skip ();
 
+    virtual std::size_t
+    position ();
+
     // Return the file path if the peeked at argument came from a file and
     // the empty string otherwise. The reference is guaranteed to be valid
     // till the end of the scanner lifetime.
diff --git a/cli/cli/options.ixx b/cli/cli/bootstrap/cli/options.ixx
index 4461340..ee4cbdb 100644
--- a/cli/cli/options.ixx
+++ b/cli/cli/bootstrap/cli/options.ixx
@@ -141,14 +141,29 @@ namespace cli
   // argv_scanner
   //
   inline argv_scanner::
-  argv_scanner (int& argc, char** argv, bool erase)
-  : i_ (1), argc_ (argc), argv_ (argv), erase_ (erase)
+  argv_scanner (int& argc,
+                char** argv,
+                bool erase,
+                std::size_t sp)
+  : start_position_ (sp + 1),
+    i_ (1),
+    argc_ (argc),
+    argv_ (argv),
+    erase_ (erase)
   {
   }
 
   inline argv_scanner::
-  argv_scanner (int start, int& argc, char** argv, bool erase)
-  : i_ (start), argc_ (argc), argv_ (argv), erase_ (erase)
+  argv_scanner (int start,
+                int& argc,
+                char** argv,
+                bool erase,
+                std::size_t sp)
+  : start_position_ (sp + static_cast<std::size_t> (start)),
+    i_ (start),
+    argc_ (argc),
+    argv_ (argv),
+    erase_ (erase)
   {
   }
 
@@ -164,8 +179,9 @@ namespace cli
   argv_file_scanner (int& argc,
                      char** argv,
                      const std::string& option,
-                     bool erase)
-  : argv_scanner (argc, argv, erase),
+                     bool erase,
+                     std::size_t sp)
+  : argv_scanner (argc, argv, erase, sp),
     option_ (option),
     options_ (&option_info_),
     options_count_ (1),
@@ -181,8 +197,9 @@ namespace cli
                      int& argc,
                      char** argv,
                      const std::string& option,
-                     bool erase)
-  : argv_scanner (start, argc, argv, erase),
+                     bool erase,
+                     std::size_t sp)
+  : argv_scanner (start, argc, argv, erase, sp),
     option_ (option),
     options_ (&option_info_),
     options_count_ (1),
@@ -195,8 +212,9 @@ namespace cli
 
   inline argv_file_scanner::
   argv_file_scanner (const std::string& file,
-                     const std::string& option)
-  : argv_scanner (0, zero_argc_, 0),
+                     const std::string& option,
+                     std::size_t sp)
+  : argv_scanner (0, zero_argc_, 0, sp),
     option_ (option),
     options_ (&option_info_),
     options_count_ (1),
@@ -214,8 +232,9 @@ namespace cli
                      char** argv,
                      const option_info* options,
                      std::size_t options_count,
-                     bool erase)
-  : argv_scanner (argc, argv, erase),
+                     bool erase,
+                     std::size_t sp)
+  : argv_scanner (argc, argv, erase, sp),
     options_ (options),
     options_count_ (options_count),
     i_ (1),
@@ -229,8 +248,9 @@ namespace cli
                      char** argv,
                      const option_info* options,
                      std::size_t options_count,
-                     bool erase)
-  : argv_scanner (start, argc, argv, erase),
+                     bool erase,
+                     std::size_t sp)
+  : argv_scanner (start, argc, argv, erase, sp),
     options_ (options),
     options_count_ (options_count),
     i_ (1),
@@ -241,8 +261,9 @@ namespace cli
   inline argv_file_scanner::
   argv_file_scanner (const std::string& file,
                      const option_info* options,
-                     std::size_t options_count)
-  : argv_scanner (0, zero_argc_, 0),
+                     std::size_t options_count,
+                     std::size_t sp)
+  : argv_scanner (0, zero_argc_, 0, sp),
     options_ (options),
     options_count_ (options_count),
     i_ (1),
diff --git a/cli/cli/buildfile b/cli/cli/buildfile
index 2385a7d..dc5d75b 100644
--- a/cli/cli/buildfile
+++ b/cli/cli/buildfile
@@ -15,63 +15,119 @@ exe{cli}:
   cli.checksum = $version
 }
 
-libue{cli}: {hxx ixx txx cxx}{** -cli -version -options -**.test...} \
-            {hxx}{version} {hxx ixx cxx}{options}                    \
-            $libs
+hdr = {hxx ixx txx}{* -cli -version -options -*.test...}
+src =           cxx{* -cli -version -options -*.test...}
+
+all_s = semantics/{hxx ixx txx cxx}{*}
+all_t = traversal/{hxx ixx txx cxx}{*}
+
+libue{cli}: $hdr $src $all_s $all_t {hxx}{version} $libs
 
 hxx{version}: in{version} $src_root/manifest
 
-# Unit tests.
+# Build options (both bootstrap and final version).
 #
-exe{*.test}:
+cxx.poptions =+ "-I$out_root" "-I$src_root"
+
+# Bootstrap.
+#
+# The plan is as follows:
+#
+# 1. Build the bootstrap version of cli using a copy of options.?xx saved in
+#    bootstrap/cli/.
+#
+# 2. Use that to re-generate options.?xx. If the result differs from the
+#    saved version, copy it over and fail the build, asking for a restart.
+#
+# 3. Otherwise, proceed to build the final version of cli.
+#
+if $config.cli.develop
 {
-  test = true
-  install = false
-}
+  libue{cli}: {hxx ixx cxx}{options}
 
-for t: cxx{**.test...}
+  bootstrap/
+  {
+    # This should only apply to the bootstrap build.
+    #
+    cxx.poptions =+ "-I$src_base" # Note: must come first.
+
+    cli/
+    {
+      # Note: semantics/ and traversal/ do not include options.hxx so we can
+      #       share their object files.
+      #
+      exe{cli}: obj{cli $name($src) options} ../../{$all_s $all_t} $libs
+
+      for s: cli $name($src)
+        obj{$s}: ../../cxx{$s} $libs
+
+      obj{options}: {hxx ixx cxx}{options}
+
+      obj{cli}: cxx.poptions += -DCLI_BOOTSTRAP
+    }
+  }
+
+  <{hxx ixx cxx}{options}>: cli{options} bootstrap/cli/exe{cli}
+  {
+    options = --include-with-brackets --include-prefix cli   \
+              --guard-prefix CLI --generate-file-scanner     \
+              --generate-specifier --generate-modifier       \
+              --suppress-undocumented --reserved-name stdout
+
+    # Symlink the generated code in src for convenience of development.
+    #
+    backlink = true
+  }
+  {{
+    diag cli ($<[0])
+    ($<[1]) $options -o $out_base $path($<[0])
+
+    # If the result differs from the bootstrap version, copy it over and
+    # request the build restart.
+    #
+    if diff $src_base/bootstrap/cli/options.hxx $path($>[0]) >- && \
+       diff $src_base/bootstrap/cli/options.ixx $path($>[1]) >- && \
+       diff $src_base/bootstrap/cli/options.cxx $path($>[2]) >-
+      exit
+    end
+
+    cp $path($>[0]) $src_base/bootstrap/cli/options.hxx
+    cp $path($>[1]) $src_base/bootstrap/cli/options.ixx
+    cp $path($>[2]) $src_base/bootstrap/cli/options.cxx
+
+    exit "bootstrap options.?xx have changed, restart the build"
+  }}
+}
+else
 {
-  d = $directory($t)
-  n = $name($t)...
+  # Use bootstrap options.?xx to build the final version of cli.
+  #
+  libue{cli}: bootstrap/cli/{hxx ixx cxx}{options}
+  cxx.poptions =+ "-I($src_base/bootstrap)" # Note: must come first.
 
-  ./: $d/exe{$n}: $t $d/{hxx ixx txx}{+$n} $d/testscript{+$n}
-  $d/exe{$n}: libue{cli}: bin.whole = false
+  ./: cli{options} # Keep in distribution.
 }
 
-# Build options.
+# Build options (final version only).
 #
+
 # Pass the copyright notice extracted from the LICENSE file.
 #
-copyright = $process.run_regex(cat $src_root/LICENSE,    \
-                               'Copyright \(c\) (.+)\.', \
-                               '\1')
-
 obj{cli}: cxx.poptions += -DCLI_COPYRIGHT=\"$copyright\"
 
-# Generated options parsing code.
-#
-# @@ This will eventually be replaced with an ah hoc recipe.
+# Unit tests.
 #
-if ($config.cli != [null] && $config.cli != false)
+exe{*.test}:
 {
-  cli.cxx{options}: cli{options}
-
-  cli.options += --include-with-brackets --include-prefix cli \
---guard-prefix CLI --generate-file-scanner --generate-specifier \
---generate-modifier --suppress-undocumented --reserved-name stdout
+  test = true
+  install = false
+}
 
-  cli.cxx{*}:
-  {
-    # Include the generated cli files into the distribution and don't remove
-    # them when cleaning in src (so that clean results in a state identical to
-    # distributed).
-    #
-    dist  = true
-    clean = ($src_root != $out_root)
+for t: cxx{**.test...}
+{
+  d = $directory($t)
+  n = $name($t)...
 
-    # We keep the generated code in the repository so copy it back to src
-    # in case of a forwarded configuration.
-    #
-    backlink = overwrite
-  }
+  ./: $d/exe{$n}: $t $d/{hxx ixx txx}{+$n} $d/testscript{+$n}
+  $d/exe{$n}: libue{cli}: bin.whole = false
 }
diff --git a/cli/cli/cli.cxx b/cli/cli/cli.cxx
index 11cf6c0..f1196de 100644
--- a/cli/cli/cli.cxx
+++ b/cli/cli/cli.cxx
@@ -16,7 +16,13 @@
 #include <cli/parser.hxx>
 #include <cli/generator.hxx>
 
-#include <cli/version.hxx>
+#ifndef CLI_BOOTSTRAP
+#  include <cli/version.hxx>
+#else
+#  define CLI_VERSION_ID 0
+#  define CLI_VERSION_FULL 0
+#  define CLI_COPYRIGHT ""
+#endif
 
 using namespace std;
 using namespace cutl;