3 files changed, 327 insertions, 0 deletions
diff --git a/unit-tests/lexer/buildfile b/unit-tests/lexer/buildfile
new file mode 100644
index 0000000..483250e
--- /dev/null
+++ b/unit-tests/lexer/buildfile
@@ -0,0 +1,12 @@
+# file      : unit-tests/lexer/buildfile
+# copyright : Copyright (c) 2009-2013 Code Synthesis Tools CC
+# license   : MIT; see accompanying LICENSE file
+
+#@@ Temporary until we get utility library support.
+#
+import libs = libcutl%lib{cutl}
+src = lexer
+
+exe{driver}: cxx{driver} ../../cli/cxx{$src} $libs test{testscript}
+
+include ../../cli/
diff --git a/unit-tests/lexer/driver.cxx b/unit-tests/lexer/driver.cxx
new file mode 100644
index 0000000..471bc1b
--- /dev/null
+++ b/unit-tests/lexer/driver.cxx
@@ -0,0 +1,123 @@
+// file      : unit-tests/lexer/driver.cxx
+// author    : Boris Kolpackov <boris@codesynthesis.com>
+// copyright : Copyright (c) 2009-2011 Code Synthesis Tools CC
+// license   : MIT; see accompanying LICENSE file
+
+#include <fstream>
+#include <iostream>
+
+#include <cli/token.hxx>
+#include <cli/lexer.hxx>
+
+using namespace std;
+
+const char* keywords[] =
+{
+  "source",
+  "include",
+  "namespace",
+  "class",
+  "signed",
+  "unsigned",
+  "bool",
+  "char",
+  "wchar_t",
+  "short",
+  "int",
+  "long",
+  "float",
+  "double"
+};
+
+const char* punctuation[] = {
+  ";", ",", ":", "::", "{", "}", /*"(", ")",*/ "=", "|"};
+
+int
+main (int argc, char* argv[])
+{
+  if (argc != 2)
+  {
+    cerr << "usage: " << argv[0] << " file.cli" << endl;
+    return 1;
+  }
+
+  ifstream ifs;
+  ifs.exceptions (ifstream::failbit | ifstream::badbit);
+  ifs.open (argv[1]);
+
+  lexer l (ifs, argv[1]);
+
+  while (true)
+  {
+    token t (l.next ());
+
+    switch (t.type ())
+    {
+    case token::t_eos:
+      {
+        cout << "<EOS>" << endl;
+        return 0;
+      }
+    case token::t_keyword:
+      {
+        cout << "keyword: " << keywords[t.keyword ()] << endl;
+        break;
+      }
+    case token::t_identifier:
+      {
+        cout << "identifier: " << t.identifier () << endl;
+        break;
+      }
+    case token::t_punctuation:
+      {
+        cout << punctuation[t.punctuation ()] << endl;
+        break;
+      }
+    case token::t_cxx_path_lit:
+      {
+        cout << "c++ path: " << t.literal () << endl;
+        break;
+      }
+    case token::t_cli_path_lit:
+      {
+        cout << "cli path: " << t.literal () << endl;
+        break;
+      }
+    case token::t_string_lit:
+      {
+        cout << t.literal () << endl;
+        break;
+      }
+    case token::t_char_lit:
+      {
+        cout << t.literal () << endl;
+        break;
+      }
+    case token::t_bool_lit:
+      {
+        cout << t.literal () << endl;
+        break;
+      }
+    case token::t_int_lit:
+      {
+        cout << t.literal () << endl;
+        break;
+      }
+    case token::t_float_lit:
+      {
+        cout << t.literal () << endl;
+        break;
+      }
+    case token::t_call_expr:
+      {
+        cout << t.expression () << endl;
+        break;
+      }
+    case token::t_template_expr:
+      {
+        cout << t.expression () << endl;
+        break;
+      }
+    }
+  }
+}
diff --git a/unit-tests/lexer/testscript b/unit-tests/lexer/testscript
new file mode 100644
index 0000000..fb4f862
--- /dev/null
+++ b/unit-tests/lexer/testscript
@@ -0,0 +1,192 @@
+# file      : unit-tests/lexer/testscript
+# copyright : Copyright (c) 2009-2013 Code Synthesis Tools CC
+# license   : MIT; see accompanying LICENSE file
+
+# @@ Give tests some meaningfull descriptions.
+#
+
+: 000
+:
+cat <<EOI >>>test.cli;
+help
+help-me
+-h
+--help
+--help-me
+--help-me-
+/h
+/help-me
+/help/me
+--_
+
+EOI
+$* test.cli >>EOO
+identifier: help
+identifier: help-me
+identifier: -h
+identifier: --help
+identifier: --help-me
+identifier: --help-me-
+identifier: /h
+identifier: /help-me
+identifier: /help
+identifier: /me
+identifier: --_
+<EOS>
+EOO
+
+: 001
+:
+cat <<EOI >>>test.cli;
+5
+123456
+-12345
+-    1
+-
+123
+EOI
+$* test.cli >>EOO
+5
+123456
+-12345
+-1
+-123
+<EOS>
+EOO
+
+: 002
+:
+cat <<EOI >>>test.cli;
+'a'
+'\\n'
+'\\\\'
+'\\0'
+'\\''
+'\\xaf'
+'\\111'
+EOI
+$* test.cli >>EOO
+'a'
+'\\n'
+'\\\\'
+'\\0'
+'\\''
+'\\xaf'
+'\\111'
+<EOS>
+EOO
+
+: 003
+:
+cat <<EOI >>>test.cli;
+"abc";
+"a\\nb";
+"abc\\\\";
+"aaa\\0";
+"\\"";
+"a\\xaf";
+"a\\111";
+"abc""def";
+"abc"   "def";
+"abc
+  def
+
+     xyz";
+EOI
+$* test.cli >>EOO
+"abc"
+;
+"a\\nb"
+;
+"abc\\\\"
+;
+"aaa\\0"
+;
+"\\""
+;
+"a\\xaf"
+;
+"a\\111"
+;
+"abc""def"
+;
+"abc""def"
+;
+"abc
+  def
+
+     xyz"
+;
+<EOS>
+EOO
+
+: 004
+:
+cat <<EOI >>>test.cli;
+include "foo/abc.hxx";
+include <vector>;
+include "c++:map";
+include <cli:map>;
+include "map.cli"
+EOI
+$* test.cli >>EOO
+keyword: include
+c++ path: "foo/abc.hxx"
+;
+keyword: include
+c++ path: <vector>
+;
+keyword: include
+c++ path: "map"
+;
+keyword: include
+cli path: <map>
+;
+keyword: include
+cli path: "map.cli"
+<EOS>
+EOO
+
+: 005
+:
+cat <<EOI >>>test.cli;
+\(abc, 123 - 345, 12.34)
+<foo, bar::baz, 123*345>
+EOI
+$* test.cli >>EOO
+\(abc, 123 - 345, 12.34)
+<foo, bar::baz, 123*345>
+<EOS>
+EOO
+
+: 006
+:
+cat <<EOI >>>test.cli;
+// c++ comment ;
+/* c comment ; */
+;
+"a" // foo
+"b"
+"a" /* foo
+bar
+baz */ "b";
+- // aaa
+5;
+- /* a
+a
+a*/ 5
+// eos
+:
+::
+EOI
+$* test.cli >>EOO
+;
+"a""b""a""b"
+;
+-5
+;
+-5
+:
+::
+<EOS>
+EOO