Implement SQLite incremental BLOB/TEXT I/O

7 files changed, 278 insertions, 6 deletions

diff --git a/odb/relational/sqlite/common.cxx b/odb/relational/sqlite/common.cxx
index 36a0e86..4ab8968 100644
--- a/odb/relational/sqlite/common.cxx
+++ b/odb/relational/sqlite/common.cxx
@@ -39,12 +39,18 @@ namespace relational
         }
       case sql_type::TEXT:
         {
-          traverse_text (mi);
+          if (mi.st->stream)
+            traverse_text_stream (mi);
+          else
+            traverse_text (mi);
           break;
         }
       case sql_type::BLOB:
         {
-          traverse_blob (mi);
+          if (mi.st->stream)
+            traverse_blob_stream (mi);
+          else
+            traverse_blob (mi);
           break;
         }
       case sql_type::invalid:
@@ -108,6 +114,12 @@ namespace relational
       type_ = "details::buffer";
     }
 
+    void member_image_type::
+    traverse_stream (member_info&)
+    {
+      type_ = "sqlite::stream_buffers";
+    }
+
     entry<member_image_type> member_image_type_;
 
     //
@@ -170,6 +182,18 @@ namespace relational
       type_id_ = "sqlite::id_blob";
     }
 
+    void member_database_type_id::
+    traverse_text_stream (member_info&)
+    {
+      type_id_ = "sqlite::id_text_stream";
+    }
+
+    void member_database_type_id::
+    traverse_blob_stream (member_info&)
+    {
+      type_id_ = "sqlite::id_blob_stream";
+    }
+
     entry<member_database_type_id> member_database_type_id_;
 
     //
diff --git a/odb/relational/sqlite/common.hxx b/odb/relational/sqlite/common.hxx
index bea0889..b5a61a1 100644
--- a/odb/relational/sqlite/common.hxx
+++ b/odb/relational/sqlite/common.hxx
@@ -56,6 +56,23 @@ namespace relational
       traverse_string (member_info&)
       {
       }
+
+      virtual void
+      traverse_text_stream (member_info& m)
+      {
+        traverse_stream (m);
+      }
+
+      virtual void
+      traverse_blob_stream (member_info& m)
+      {
+        traverse_stream (m);
+      }
+
+      virtual void
+      traverse_stream (member_info&)
+      {
+      }
     };
 
     struct member_image_type: relational::member_image_type,
@@ -82,6 +99,9 @@ namespace relational
       virtual void
       traverse_string (member_info&);
 
+      virtual void
+      traverse_stream (member_info&);
+
     private:
       string type_;
     };
@@ -114,6 +134,12 @@ namespace relational
       virtual void
       traverse_blob (member_info&);
 
+      virtual void
+      traverse_text_stream (member_info&);
+
+      virtual void
+      traverse_blob_stream (member_info&);
+
     private:
       string type_id_;
     };
diff --git a/odb/relational/sqlite/context.cxx b/odb/relational/sqlite/context.cxx
index 007d67f..94b8067 100644
--- a/odb/relational/sqlite/context.cxx
+++ b/odb/relational/sqlite/context.cxx
@@ -339,10 +339,23 @@ namespace relational
           if (ids_.empty ())
             return error ("expected SQLite type name");
 
+          // First check our own types.
+          //
+          if (ids_.size () == 2 && ids_[0] == "TEXT" && ids_[1] == "STREAM")
+          {
+            r.type = sql_type::TEXT;
+            r.stream = true;
+          }
+          if (ids_.size () == 2 && ids_[0] == "BLOB" && ids_[1] == "STREAM")
+          {
+            r.type = sql_type::BLOB;
+            r.stream = true;
+          }
+          //
           // Apply the first four rules of the SQLite type to affinity
           // conversion algorithm.
           //
-          if (find ("INT"))
+          else if (find ("INT"))
             r.type = sql_type::INTEGER;
           else if (find ("TEXT") || find ("CHAR") || find ("CLOB"))
             r.type = sql_type::TEXT;
diff --git a/odb/relational/sqlite/context.hxx b/odb/relational/sqlite/context.hxx
index 7c5107e..6fcdc47 100644
--- a/odb/relational/sqlite/context.hxx
+++ b/odb/relational/sqlite/context.hxx
@@ -26,9 +26,10 @@ namespace relational
         invalid
       };
 
-      sql_type (): type (invalid) {}
+      sql_type (): type (invalid), stream (false) {}
 
       core_type type;
+      bool stream; // TEXT or BLOB via sqlite3_blob_open().
 
       // Conversion expressions for custom database types.
       //
diff --git a/odb/relational/sqlite/header.cxx b/odb/relational/sqlite/header.cxx
index 9432825..e548413 100644
--- a/odb/relational/sqlite/header.cxx
+++ b/odb/relational/sqlite/header.cxx
@@ -48,6 +48,15 @@ namespace relational
              << "bool " << mi.var << "null;"
              << endl;
         }
+
+        virtual void
+        traverse_stream (member_info& mi)
+        {
+          os << image_type << " " << mi.var << "value;"
+             << "std::size_t " << mi.var << "size;"
+             << "bool " << mi.var << "null;"
+             << endl;
+        }
       };
       entry<image_member> image_member_;
     }
diff --git a/odb/relational/sqlite/model.cxx b/odb/relational/sqlite/model.cxx
index 47eb88b..45216b7 100644
--- a/odb/relational/sqlite/model.cxx
+++ b/odb/relational/sqlite/model.cxx
@@ -23,6 +23,27 @@ namespace relational
       {
         object_columns (base const& x): base (x) {}
 
+        virtual string
+        type (semantics::data_member& m)
+        {
+          // Translate BLOB|TEXT STREAM to just BLOB|TEXT.
+          //
+          string r (relational::object_columns::type (m));
+
+          sql_type const& t (parse_sql_type (r, m, false));
+          if (t.stream)
+          {
+            switch (t.type)
+            {
+            case sql_type::BLOB: r = "BLOB"; break;
+            case sql_type::TEXT: r = "TEXT"; break;
+            default: break;
+            }
+          }
+
+          return r;
+        }
+
         virtual bool
         null (semantics::data_member& m)
         {
diff --git a/odb/relational/sqlite/source.cxx b/odb/relational/sqlite/source.cxx
index 0d821ac..2f6623e 100644
--- a/odb/relational/sqlite/source.cxx
+++ b/odb/relational/sqlite/source.cxx
@@ -17,6 +17,72 @@ namespace relational
     {
       namespace relational = relational::source;
 
+      struct query_parameters: relational::query_parameters, context
+      {
+        query_parameters (base const& x): base (x) {}
+
+        virtual string
+        next (semantics::data_member& m,
+              const string& column,
+              const string& sqlt)
+        {
+          // Handle stream columns. Specifically, we somehow need to
+          // pass the column name to the code that runs in the
+          // statement. So what we are going to do is encode it
+          // in the parameter name.
+          //
+          if (sk_ == statement_insert || sk_ == statement_update)
+          {
+            const sql_type& t (parse_sql_type (sqlt, m, false));
+            if (t.stream)
+            {
+              // The column name is quoted.
+              //
+              string r (column);
+              r[0] = '$';               // Replace leading '"'.
+              r.resize (r.size () - 1); // Remove trailing '"'.
+
+              // Verify it only contains allowed characters.
+              //
+              for (size_t i (1); i != r.size (); ++i)
+              {
+                char c (r[i]);
+                if (c != '_' &&
+                    (c < '0' || c > '9') &&
+                    (c < 'a' || c > 'z') &&
+                    (c < 'A' || c > 'Z'))
+                {
+                  cerr << m.file () << ":" << m.line () << ":" << m.column ()
+                       << ": error: unsupported character '" << c << "' in "
+                       << sqlt << " column name " << column << endl;
+
+                  cerr << m.file () << ":" << m.line () << ":" << m.column ()
+                       << ": info: STREAM column can contain alpha-numeric "
+                       << "characters plus '_'" << endl;
+
+                  throw operation_failed ();
+                }
+              }
+
+              // For TEXT columns, since we use the *_bind_zeroblob()
+              // function (there is no *_bind_zerotext()), the value
+              // that will be stored is BLOB, not TEXT, unless we
+              // explicitly CAST it. The user better make sure the
+              // encoding of raw TEXT data they are going to write
+              // matches the database encoding.
+              //
+              if (t.type == sql_type::TEXT)
+                r = "CAST(" + r + " AS TEXT)";
+
+              return r;
+            }
+          }
+
+          return "?";
+        }
+      };
+      entry<query_parameters> query_parameters_;
+
       //
       // bind
       //
@@ -71,6 +137,15 @@ namespace relational
             "value.capacity ();"
              << b << ".is_null = &" << arg << "." << mi.var << "null;";
         }
+
+        virtual void
+        traverse_stream (member_info& mi)
+        {
+          os << b << ".type = sqlite::bind::stream;"
+             << b << ".buffer = &" << arg << "." << mi.var << "value;"
+             << b << ".size = &" << arg << "." << mi.var << "size;"
+             << b << ".is_null = &" << arg << "." << mi.var << "null;";
+        }
       };
       entry<bind_member> bind_member_;
 
@@ -110,6 +185,13 @@ namespace relational
              << "grew = true;"
              << "}";
         }
+
+        virtual void
+        traverse_stream (member_info&)
+        {
+          os << e << " = false;"
+             << endl;
+        }
       };
       entry<grow_member> grow_member_;
 
@@ -166,6 +248,17 @@ namespace relational
              << "i." << mi.var << "null = is_null;"
              << "grew = grew || (cap != i." << mi.var << "value.capacity ());";
         }
+
+        virtual void
+        traverse_stream (member_info& mi)
+        {
+          os << traits << "::set_image (" << endl
+             << "i." << mi.var << "value," << endl
+             << "i." << mi.var << "size," << endl
+             << "is_null," << endl
+             << member << ");"
+             << "i." << mi.var << "null = is_null;";
+        }
       };
       entry<init_image_member> init_image_member_;
 
@@ -220,10 +313,65 @@ namespace relational
              << "i." << mi.var << "null);"
              << endl;
         }
+
+        virtual void
+        traverse_stream (member_info& mi)
+        {
+          os << traits << "::set_value (" << endl
+             << member << "," << endl
+             << "i." << mi.var << "value," << endl
+             << "i." << mi.var << "size," << endl
+             << "i." << mi.var << "null);"
+             << endl;
+        }
       };
       entry<init_value_member> init_value_member_;
 
-      struct container_traits: relational::container_traits, context
+      struct statement_columns_common: context
+      {
+        void
+        process (relational::statement_columns& cs, statement_kind sk)
+        {
+          using relational::statement_columns;
+
+          // For SELECT statements, add _ROWID_ "follow-up" column to
+          // each stream column. The reason we need both, and not just
+          // ROWID is the NULL value. Let's hope that SELECT'ing a BLOB
+          // but not actually reading it with sqlite3_result_blob() is
+          // as fast as not SELECT'ing it.
+          //
+          if (sk != statement_select)
+            return;
+
+          for (statement_columns::iterator i (cs.begin ());
+               i != cs.end (); ++i)
+          {
+            if (parse_sql_type (i->type, *i->member).stream)
+            {
+              // Column is already table-qualified and quoted. Do some
+              // surgery to replace it with _ROWID_. That is, we want to
+              // transform "table"."column" to "table"."_ROWID_".
+              //
+              string c (i->column);
+              string::size_type n (c.size ()), p (c.rfind ('"', n - 2));
+              assert (p != string::npos);
+              string as (c, p + 1, n - p - 2);
+              c.resize (p);
+              c += "\"_ROWID_\"";
+
+              // We are going to pack this "tightly", without any newlines,
+              // so that the statement processing code treats them as a
+              // single column.
+              //
+              i->column += ',';
+              i->column += c;
+            }
+          }
+        }
+      };
+
+      struct container_traits: relational::container_traits,
+                               statement_columns_common
       {
         container_traits (base const& x): base (x) {}
 
@@ -234,10 +382,32 @@ namespace relational
           // interleaving statements.
           //
         }
+
+        virtual void
+        process_statement_columns (relational::statement_columns& cols,
+                                   statement_kind sk,
+                                   bool)
+        {
+          statement_columns_common::process (cols, sk);
+        }
       };
       entry<container_traits> container_traits_;
 
-      struct class_: relational::class_, context
+      struct section_traits: relational::section_traits,
+                             statement_columns_common
+      {
+        section_traits (base const& x): base (x) {}
+
+        virtual void
+        process_statement_columns (relational::statement_columns& cols,
+                                   statement_kind sk)
+        {
+          statement_columns_common::process (cols, sk);
+        }
+      };
+      entry<section_traits> section_traits_;
+
+      struct class_: relational::class_, statement_columns_common
       {
         class_ (base const& x): base (x) {}
 
@@ -274,6 +444,14 @@ namespace relational
 
           return base::join_syntax (vo);
         }
+
+        virtual void
+        process_statement_columns (relational::statement_columns& cols,
+                                   statement_kind sk,
+                                   bool)
+        {
+          statement_columns_common::process (cols, sk);
+        }
       };
       entry<class_> class_entry_;
     }