From 8f6a9c51bc64226d7c296e4b0172f9e56a7eea3b Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Wed, 15 Jul 2015 18:43:03 +0200 Subject: Implement SQLite incremental BLOB/TEXT I/O --- NEWS | 4 + doc/manual.xhtml | 358 ++++++++++++++++++++++++++++++++++++++ odb/relational/context.hxx | 1 + odb/relational/model.hxx | 9 +- odb/relational/mssql/source.cxx | 2 +- odb/relational/oracle/source.cxx | 16 +- odb/relational/pgsql/source.cxx | 4 +- odb/relational/source.cxx | 36 ++-- odb/relational/source.hxx | 82 ++++++--- odb/relational/sqlite/common.cxx | 28 ++- odb/relational/sqlite/common.hxx | 26 +++ odb/relational/sqlite/context.cxx | 15 +- odb/relational/sqlite/context.hxx | 3 +- odb/relational/sqlite/header.cxx | 9 + odb/relational/sqlite/model.cxx | 21 +++ odb/relational/sqlite/source.cxx | 182 ++++++++++++++++++- 16 files changed, 738 insertions(+), 58 deletions(-) diff --git a/NEWS b/NEWS index 030073f..1df084d 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,10 @@ Version 2.5.0 * Support for defining views as instantiations of C++ class templates, similar to objects and composite value types. + * Support for SQLite incremental BLOB/TEXT I/O (the sqlite3_blob_open() + functionality). For details, refer to Section 18.1.3, "Incremental + BLOB/TEXT I/O" in the ODB manual. + Version 2.4.0 * Support for object loading views. Object loading views allow loading of diff --git a/doc/manual.xhtml b/doc/manual.xhtml index 7b86be9..5c28564 100644 --- a/doc/manual.xhtml +++ b/doc/manual.xhtml @@ -726,6 +726,7 @@ for consistency. +
18.1.1String Type Mapping
18.1.2Binary Type Mapping
18.1.3Incremental BLOB/TEXT I/O
@@ -20829,6 +20830,18 @@ t.commit (); TEXT NOT NULL + + + odb::sqlite::text + TEXT (STREAM) + NOT NULL + + + + odb::sqlite::blob + BLOB (STREAM) + NOT NULL +

It is possible to map the char C++ type to the @@ -20983,6 +20996,351 @@ db.query<object> ("uuid = " + query::_val<odb::sqlite::id_blob> (u)); db.query<object> (query::uuid == query::_ref (u)); +

18.1.3 Incremental BLOB/TEXT I/O

+ +

This section describes the SQLite ODB runtime library support for + incremental reading and writing of BLOB and + TEXT values. The provided API is a thin wrapper + around the native SQLite sqlite3_blob_*() function + family. As a result, it is highly recommended that you familiarize + yourself with the semantics of this SQLite functionality before + continuing with this section.

+ +

The SQLite runtime provides the blob and + text types that can be used to represent + BLOB and TEXT data members + that will be read/written using the incremental I/O. + For example:

+ + 
+#include <odb/sqlite/blob.hxx>
+#include <odb/sqlite/text.hxx>
+
+#pragma db object
+class object
+{
+public
+  #pragma db id auto
+  unsigned long id;
+
+  odb::sqlite::blob b; // Mapped to BLOB.
+  odb::sqlite::text t; // Mapped to TEXT.
+};
+
+ +

The blob and text types should be + viewed as descriptors of the BLOB and + TEXT values (rather than the values themselves) + that can be used to open the values for reading or + writing. These two types have an identical interface that + is presented below. Notice that it is essentially the list + of arguments (except for size which is discussed + below) to the sqlite3_blob_open() function:

+ + 
+namespace odb
+{
+  namespace sqlite
+  {
+    class blob|text
+    {
+    public:
+      explicit
+      blob|text (std::size_t = 0);
+
+      std::size_t size ()
+      void        size (std::size_t);
+
+      const std::string& db () const;
+      const std::string& table () const;
+      const std::string& column () const;
+      long long          rowid () const;
+
+      void
+      clear ();
+    };
+  }
+}
+
+ +

To read/write data from/to a incremental BLOB or + TEXT value we use the corresponding + blob_stream and text_stream + stream types. Their interfaces closely mimic the + underlying sqlite3_blob_*() functions + and are presented below. Note that in order to create + a stream we have to pass the corresponding descriptor:

+ + 
+#include <odb/sqlite/stream.hxx>
+
+namespace odb
+{
+  namespace sqlite
+  {
+    class stream
+    {
+    public:
+      stream (const char* db,
+              const char* table,
+              const char* column,
+              long long rowid,
+              bool rw);
+
+      std::size_t
+      size () const;
+
+      // The following two functions throw std::invalid_argument if
+      // offset + n is past size().
+      //
+      void
+      read (void* buf, std::size_t n, std::size_t offset = 0);
+
+      void
+      write (const void* buf, std::size_t n, std::size_t offset = 0);
+
+      sqlite3_blob*
+      handle () const;
+
+      // Close without reporting errors, if any.
+      //
+      ~stream ();
+
+      // Close with reporting errors, if any.
+      //
+      void
+      close ();
+
+      // Open the same BLOB but in a different row. Can be faster
+      // than creating a new stream instance. Note that the stream
+      // must be in the open state prior to calling this function.
+      //
+      void
+      reopen (long long rowid);
+    };
+  }
+}
+
+#include <odb/sqlite/blob-stream.hxx>
+
+namespace odb
+{
+  namespace sqlite
+  {
+    class blob_stream: public stream
+    {
+    public:
+      blob_stream (const blob&, bool rw);
+    };
+  }
+}
+
+#include <odb/sqlite/text-stream.hxx>
+
+namespace odb
+{
+  namespace sqlite
+  {
+    class text_stream: public stream
+    {
+    public:
+      text_stream (const text&, bool rw);
+    };
+  }
+}
+
+ +

The rw argument to the constructors above + specifies whether to open the value for reading only + (false) or to read and write + (true).

+ +

In SQLite the incremental BLOB and + TEXT sizes are fixed in the sense that + they must be specified before the object is persisted + or updated and the following write operations can + only write that much data. This is what the size + data member in the descriptors is for. You can also determine + the size of the opened value, for both reading and writing, + using the size() stream function. The + following example puts all of this together:

+ + 
+#include <odb/sqlite/blob-stream.hxx>
+#include <odb/sqlite/text-stream.hxx>
+
+string txt (1024 * 1024, 't');
+vector<char> blb (1024 * 1024, 'b');
+
+object o;
+
+// Persist.
+//
+{
+  transaction tx (db.begin ());
+
+  // Specify the sizes of the values before calling persist().
+  //
+  o.t.size (txt.size ());
+  o.b.size (blb.size ());
+
+  db.persist (o);
+
+  // Write the data.
+  //
+  blob_stream bs (o.b, true); // Open for read/write.
+  assert (bs.size () == blb.size ());
+  bs.write (blb.data (), blb.size ());
+
+  text_stream ts (o.t, true); // Open for read/write.
+  assert (ts.size () == txt.size ());
+  ts.write (txt.data (), txt.size ());
+
+  tx.commit ();
+}
+
+// Load.
+//
+{
+  transaction tx (db.begin ());
+  auto_ptr<object> p (db.load<object> (o.id));
+
+  text_stream ts (p->t, false); // Open for reading.
+  vector<char> t (ts.size () + 1, '\0');
+  ts.read (t.data (), t.size () - 1);
+  assert (string (t.data ()) == txt);
+
+  blob_stream bs (p->b, false); // Open for reading.
+  vector<char> b (bs.size (), '\0');
+  bs.read (b.data (), b.size ());
+  assert (b == blb);
+
+  tx.commit ();
+}
+
+// Update
+//
+txt.resize (txt.size () + 1, 't');
+txt[0] = 'A';
+txt[txt.size () - 1] = 'Z';
+
+blb.resize (blb.size () - 1);
+blb.front () = 'A';
+blb.back () = 'Z';
+
+{
+  transaction tx (db.begin ());
+
+  // Specify the new sizes of the values before calling update().
+  //
+  o.t.size (txt.size ());
+  o.b.size (blb.size ());
+
+  db.update (o);
+
+  // Write the data.
+  //
+  blob_stream bs (o.b, true);
+  bs.write (blb.data (), blb.size ());
+
+  text_stream ts (o.t, true);
+  ts.write (txt.data (), txt.size ());
+
+  tx.commit ();
+}
+
+ +

For the most part, the incremental BLOB and + TEXT descriptors can be used as any other + simple values. Specifically, they can be used as container + elements (Chapter 5, "Containers"), as + NULL-able values (Section 7.3, + "Pointers and NULL Value Semantics"), and in views + (Chapter 10, "Views"). The following + example illustrates the use within a view:

+ + 
+#pragma db view object(object)
+struct load_b
+{
+  odb::sqlite::blob b;
+};
+
+typedef odb::query<load_b> query;
+
+transaction tx (db.begin ());
+
+for (load_b& lb: db.query<load_b> (query::t == "test"))
+{
+  blob_stream bs (lb.b, false);
+  vector<char> b (bs.size (), '\0');
+  bs.read (b.data (), b.size ());
+}
+
+tx.commit ();
+
+ +

However, being a low-level, SQLite-specific mechanism, the + incremental I/O has a number of nuances that should be kept in + mind. Firstly, the streams should be opened within a transaction + and, unless already closed, they will be automatically closed + when the transaction is committed or rolled back. The following + modification of the previous example helps to illustrate this + point:

+ + 
+{
+  transaction tx (db.begin ());
+
+  // ...
+
+  db.persist (o);
+
+  blob_stream bs (o.b, true);
+
+  tx.commit ();
+
+  // ERROR: stream is closed.
+  //
+  bs.write (blb.data (), blb.size ());
+}
+
+// ERROR: not in transaction.
+//
+text_stream ts (o.t, true);
+
+ +

Because loading an object with an incremental BLOB or + TEXT value involves additional actions after the + database function returns (that is, reading the actual data), + certain commonly-expected "round-trip" assumptions will no + longer hold unless special steps are taken, for instance + (again, continuing with our example):

+ + 
+transaction tx (db.begin ());
+
+auto_ptr<object> p (db.load<object> (o.id));
+p->name = "foo"; // Update some other member.
+db.update (*p);  // Bad behavior: incremental BLOB/TEXT invalidated.
+
+tx.commit ();
+
+ +

One way to restore the expected behavior is to place the + incremental BLOB and TEXT values + into their own, separately loaded/updated sections + (Chapter 9, "Sections"). The alternative + approach would be to perform the incremental I/O as part + of the database operation post_* callbacks + (Section 14.1.7, "callback").

+ +

Finally, note that when using incremental TEXT + values, the data that we read/write is the raw bytes in + the encoding used by the database (UTF-8 by + default; see SQLite PRAGMA encoding documentation + for details).

+

18.2 SQLite Database Class

The SQLite database class has the following diff --git a/odb/relational/context.hxx b/odb/relational/context.hxx index 02adfc7..b172e45 100644 --- a/odb/relational/context.hxx +++ b/odb/relational/context.hxx @@ -20,6 +20,7 @@ namespace relational statement_select, statement_insert, statement_update, + statement_delete, statement_where // WHERE clause. }; diff --git a/odb/relational/model.hxx b/odb/relational/model.hxx index 13c67d7..3b53c67 100644 --- a/odb/relational/model.hxx +++ b/odb/relational/model.hxx @@ -115,8 +115,7 @@ namespace relational (key_prefix_.empty () ? m.name () : key_prefix_)); sema_rel::column& c ( - model_.new_node ( - col_id, column_type (), null (m))); + model_.new_node (col_id, type (m), null (m))); c.set ("cxx-location", m.location ()); c.set ("member-path", member_path_); model_.new_edge (table_, c, name); @@ -142,6 +141,12 @@ namespace relational return true; } + virtual string + type (semantics::data_member&) + { + return object_columns_base::column_type (); + } + virtual bool null (semantics::data_member&) { diff --git a/odb/relational/mssql/source.cxx b/odb/relational/mssql/source.cxx index 42c29a7..4384761 100644 --- a/odb/relational/mssql/source.cxx +++ b/odb/relational/mssql/source.cxx @@ -24,7 +24,7 @@ namespace relational query_parameters (base const& x): base (x) {} virtual string - auto_id () + auto_id (semantics::data_member&, const string&, const string&) { return ""; } diff --git a/odb/relational/oracle/source.cxx b/odb/relational/oracle/source.cxx index 802bc99..b69b291 100644 --- a/odb/relational/oracle/source.cxx +++ b/odb/relational/oracle/source.cxx @@ -22,7 +22,7 @@ namespace relational query_parameters (base const& x): base (x), i_ (0) {} virtual string - next () + next (semantics::data_member&, const string&, const string&) { ostringstream ss; ss << ":" << ++i_; @@ -31,7 +31,7 @@ namespace relational } virtual string - auto_id () + auto_id (semantics::data_member&, const string&, const string&) { return quote_id (sequence_name (table_)) + ".nextval"; } @@ -612,9 +612,15 @@ namespace relational // Top-level auto id. // if (id != 0 && !poly_derived && auto_ (*id)) - r = "RETURNING " + - convert_from (column_qname (*id), *id->back ()) + - " INTO " + qp.next (); + { + semantics::data_member& idb (*id->back ()); + + const string& name (column_qname (*id)); + const string& type (column_type (idb)); + + r = "RETURNING " + convert_from (name, type, idb) + + " INTO " + qp.next (idb, name, type); + } } return r; diff --git a/odb/relational/pgsql/source.cxx b/odb/relational/pgsql/source.cxx index b8270ac..b0155b7 100644 --- a/odb/relational/pgsql/source.cxx +++ b/odb/relational/pgsql/source.cxx @@ -24,7 +24,7 @@ namespace relational query_parameters (base const& x): base (x), i_ (0) {} virtual string - next () + next (semantics::data_member&, const string&, const string&) { ostringstream ss; ss << "$" << ++i_; @@ -33,7 +33,7 @@ namespace relational } virtual string - auto_id () + auto_id (semantics::data_member&, const string&, const string&) { return "DEFAULT"; } diff --git a/odb/relational/source.cxx b/odb/relational/source.cxx index 3b96ab5..e9ddd77 100644 --- a/odb/relational/source.cxx +++ b/odb/relational/source.cxx @@ -746,7 +746,7 @@ traverse_object (type& c) os << strlit (s) << endl; } - instance qp (table); + instance qp (statement_insert, table); string extra (persist_statement_extra (c, *qp, persist_after_columns)); @@ -854,7 +854,7 @@ traverse_object (type& c) } string where ("WHERE "); - instance qp (table); + instance qp (statement_select, table); for (object_columns_list::iterator b (id_cols->begin ()), i (b); i != id_cols->end (); ++i) { @@ -862,7 +862,7 @@ traverse_object (type& c) where += " AND "; where += qtable + "." + quote_id (i->name) + "=" + - convert_to (qp->next (), i->type, *i->member); + convert_to (qp->next (*i), i->type, *i->member); } os << strlit (where); @@ -936,7 +936,7 @@ traverse_object (type& c) os << strlit ("FROM " + qtable + " ") << endl; string where ("WHERE "); - instance qp (table); + instance qp (statement_select, table); for (object_columns_list::iterator b (id_cols->begin ()), i (b); i != id_cols->end (); ++i) { @@ -944,7 +944,7 @@ traverse_object (type& c) where += " AND "; where += qtable + "." + quote_id (i->name) + "=" + - convert_to (qp->next (), i->type, *i->member); + convert_to (qp->next (*i), i->type, *i->member); } os << strlit (where) << ";" @@ -957,7 +957,7 @@ traverse_object (type& c) { string sep (versioned ? "\n" : " "); - instance qp (table); + instance qp (statement_update, table); statement_columns sc; { @@ -1005,15 +1005,18 @@ traverse_object (type& c) where += " AND "; where += quote_id (i->name) + "=" + - convert_to (qp->next (), i->type, *i->member); + convert_to (qp->next (*i), i->type, *i->member); } // Top-level version column. // if (opt != 0 && !poly_derived) { - where += " AND " + column_qname (*opt, column_prefix ()) + "=" + - convert_to (qp->next (), *opt); + string name (column_qname (*opt, column_prefix ())); + string type (column_type (*opt)); + + where += " AND " + name + "=" + + convert_to (qp->next (*opt, name, type), type, *opt); } os << strlit (where) << ";" @@ -1023,7 +1026,7 @@ traverse_object (type& c) // erase_statement // { - instance qp (table); + instance qp (statement_delete, table); os << "const char " << traits << "::erase_statement[] =" << endl << strlit ("DELETE FROM " + qtable + " ") << endl; @@ -1035,7 +1038,7 @@ traverse_object (type& c) where += " AND "; where += quote_id (i->name) + "=" + - convert_to (qp->next (), i->type, *i->member); + convert_to (qp->next (*i), i->type, *i->member); } os << strlit (where) << ";" @@ -1044,7 +1047,7 @@ traverse_object (type& c) if (opt != 0 && !poly_derived) { - instance qp (table); + instance qp (statement_delete, table); os << "const char " << traits << "::optimistic_erase_statement[] " << "=" << endl @@ -1058,13 +1061,16 @@ traverse_object (type& c) where += " AND "; where += quote_id (i->name) + "=" + - convert_to (qp->next (), i->type, *i->member); + convert_to (qp->next (*i), i->type, *i->member); } // Top-level version column. // - where += " AND " + column_qname (*opt, column_prefix ()) + "=" + - convert_to (qp->next (), *opt); + string name (column_qname (*opt, column_prefix ())); + string type (column_type (*opt)); + + where += " AND " + name + "=" + + convert_to (qp->next (*opt, name, type), type, *opt); os << strlit (where) << ";" << endl; diff --git a/odb/relational/source.hxx b/odb/relational/source.hxx index f495cef..9b23b31 100644 --- a/odb/relational/source.hxx +++ b/odb/relational/source.hxx @@ -55,21 +55,39 @@ namespace relational { typedef query_parameters base; - query_parameters (qname const& table): table_ (table) {} + query_parameters (statement_kind sk, qname const& table) + : sk_ (sk), table_ (table) {} virtual string - next () + next (semantics::data_member&, + const std::string& /*column*/, // Table qualified and quoted. + const std::string& /*sqlt*/) { return "?"; } virtual string - auto_id () + auto_id (semantics::data_member& m, + const std::string& column, + const std::string& sqlt) { - return next (); + return next (m, column, sqlt); + } + + string + next (const object_columns_list::column& c) + { + return next (*c.member, quote_id (c.name), c.type); + } + + string + next (const statement_column& c) + { + return next (*c.member, c.column, c.type); } protected: + statement_kind sk_; qname table_; }; @@ -338,7 +356,7 @@ namespace relational else if (param_ != 0) { r += '='; - r += convert_to (param_->next (), sqlt, m); + r += convert_to (param_->next (m, column, sqlt), sqlt, m); } else if (sk_ == statement_select) r = convert_from (r, sqlt, m); @@ -3634,7 +3652,7 @@ namespace relational os << strlit (c + (++i != e ? "," : "") + sep) << endl; } - instance qp (inv_table); + instance qp (statement_select, inv_table); os << strlit ("FROM " + inv_qtable + sep) << endl; string where ("WHERE "); @@ -3645,7 +3663,7 @@ namespace relational where += " AND "; where += inv_qtable + "." + quote_id (i->name) + "=" + - convert_to (qp->next (), i->type, *i->member); + convert_to (qp->next (*i), i->type, *i->member); } os << strlit (where); } @@ -3691,7 +3709,7 @@ namespace relational os << strlit (c + (++i != e ? "," : "") + sep) << endl; } - instance qp (table); + instance qp (statement_select, table); os << strlit ("FROM " + qtable + sep) << endl; string where ("WHERE "); @@ -3702,7 +3720,7 @@ namespace relational where += " AND "; where += qtable + "." + quote_id (i->name) + "=" + - convert_to (qp->next (), i->type, *i->member); + convert_to (qp->next (*i), i->type, *i->member); } if (ordered) @@ -3781,7 +3799,7 @@ namespace relational os << strlit ("VALUES" + sep) << endl; string values ("("); - instance qp (table); + instance qp (statement_insert, table); for (statement_columns::const_iterator b (sc.begin ()), i (b), e (sc.end ()); i != e; ++i) { @@ -3791,7 +3809,7 @@ namespace relational values += sep; } - values += convert_to (qp->next (), i->type, *i->member); + values += convert_to (qp->next (*i), i->type, *i->member); } values += ')'; @@ -3808,7 +3826,7 @@ namespace relational << strlit ("UPDATE " + qtable + sep) << endl << strlit ("SET" + sep) << endl; - instance qp (table); + instance qp (statement_update, table); statement_columns sc; { bool f (false); // Imperfect forwarding. @@ -3834,14 +3852,14 @@ namespace relational where += " AND "; where += quote_id (i->name) + "=" + - convert_to (qp->next (), i->type, *i->member); + convert_to (qp->next (*i), i->type, *i->member); } for (object_columns_list::iterator b (ik_cols->begin ()), i (b); i != ik_cols->end (); ++i) { where += " AND " + quote_id (i->name) + "=" + - convert_to (qp->next (), i->type, *i->member); + convert_to (qp->next (*i), i->type, *i->member); } os << strlit (where) << ";" @@ -3858,7 +3876,7 @@ namespace relational << endl; else { - instance qp (table); + instance qp (statement_delete, table); os << strlit ("DELETE FROM " + qtable + " ") << endl; @@ -3870,7 +3888,7 @@ namespace relational where += " AND "; where += quote_id (i->name) + "=" + - convert_to (qp->next (), i->type, *i->member); + convert_to (qp->next (*i), i->type, *i->member); } if (smart) @@ -3880,7 +3898,7 @@ namespace relational { where += " AND " + quote_id (i->name) + (ck == ck_ordered ? ">=" : "=") + - convert_to (qp->next (), i->type, *i->member); + convert_to (qp->next (*i), i->type, *i->member); } } @@ -5961,7 +5979,7 @@ namespace relational } string where ("WHERE "); - instance qp (table); + instance qp (statement_select, table); for (object_columns_list::iterator b (id_cols->begin ()), i (b); i != id_cols->end (); ++i) { @@ -5969,7 +5987,7 @@ namespace relational where += " AND "; where += qtable + "." + quote_id (i->name) + "=" + - convert_to (qp->next (), i->type, *i->member); + convert_to (qp->next (*i), i->type, *i->member); } os << strlit (where) << ";" @@ -5980,7 +5998,7 @@ namespace relational // if (update || update_opt) { - instance qp (table); + instance qp (statement_update, table); statement_columns sc; { @@ -6029,13 +6047,16 @@ namespace relational where += " AND "; where += quote_id (i->name) + "=" + - convert_to (qp->next (), i->type, *i->member); + convert_to (qp->next (*i), i->type, *i->member); } if (s.optimistic ()) // Note: not update_opt. { - where += " AND " + column_qname (*opt, column_prefix ()) + "=" + - convert_to (qp->next (), *opt); + string name (column_qname (*opt, column_prefix ())); + string type (column_type (*opt)); + + where += " AND " + name + "=" + + convert_to (qp->next (*opt, name, type), type, *opt); } os << strlit (where) << ";" @@ -6450,16 +6471,23 @@ namespace relational } virtual bool - traverse_column (semantics::data_member& m, string const&, bool first) + traverse_column (semantics::data_member& m, + string const& name, + bool first) { string p; if (version (m)) p = version_value (m); - else if (auto_ (m)) // Only simple, direct id can be auto. - p = qp_.auto_id (); else - p = qp_.next (); + { + const string& qname (quote_id (name)); + const string& type (column_type ()); + + p = auto_ (m) // Only simple, direct id can be auto. + ? qp_.auto_id (m, qname, type) + : qp_.next (m, qname, type); + } if (!p.empty ()) { diff --git a/odb/relational/sqlite/common.cxx b/odb/relational/sqlite/common.cxx index 36a0e86..4ab8968 100644 --- a/odb/relational/sqlite/common.cxx +++ b/odb/relational/sqlite/common.cxx @@ -39,12 +39,18 @@ namespace relational } case sql_type::TEXT: { - traverse_text (mi); + if (mi.st->stream) + traverse_text_stream (mi); + else + traverse_text (mi); break; } case sql_type::BLOB: { - traverse_blob (mi); + if (mi.st->stream) + traverse_blob_stream (mi); + else + traverse_blob (mi); break; } case sql_type::invalid: @@ -108,6 +114,12 @@ namespace relational type_ = "details::buffer"; } + void member_image_type:: + traverse_stream (member_info&) + { + type_ = "sqlite::stream_buffers"; + } + entry member_image_type_; // @@ -170,6 +182,18 @@ namespace relational type_id_ = "sqlite::id_blob"; } + void member_database_type_id:: + traverse_text_stream (member_info&) + { + type_id_ = "sqlite::id_text_stream"; + } + + void member_database_type_id:: + traverse_blob_stream (member_info&) + { + type_id_ = "sqlite::id_blob_stream"; + } + entry member_database_type_id_; // diff --git a/odb/relational/sqlite/common.hxx b/odb/relational/sqlite/common.hxx index bea0889..b5a61a1 100644 --- a/odb/relational/sqlite/common.hxx +++ b/odb/relational/sqlite/common.hxx @@ -56,6 +56,23 @@ namespace relational traverse_string (member_info&) { } + + virtual void + traverse_text_stream (member_info& m) + { + traverse_stream (m); + } + + virtual void + traverse_blob_stream (member_info& m) + { + traverse_stream (m); + } + + virtual void + traverse_stream (member_info&) + { + } }; struct member_image_type: relational::member_image_type, @@ -82,6 +99,9 @@ namespace relational virtual void traverse_string (member_info&); + virtual void + traverse_stream (member_info&); + private: string type_; }; @@ -114,6 +134,12 @@ namespace relational virtual void traverse_blob (member_info&); + virtual void + traverse_text_stream (member_info&); + + virtual void + traverse_blob_stream (member_info&); + private: string type_id_; }; diff --git a/odb/relational/sqlite/context.cxx b/odb/relational/sqlite/context.cxx index 007d67f..94b8067 100644 --- a/odb/relational/sqlite/context.cxx +++ b/odb/relational/sqlite/context.cxx @@ -339,10 +339,23 @@ namespace relational if (ids_.empty ()) return error ("expected SQLite type name"); + // First check our own types. + // + if (ids_.size () == 2 && ids_[0] == "TEXT" && ids_[1] == "STREAM") + { + r.type = sql_type::TEXT; + r.stream = true; + } + if (ids_.size () == 2 && ids_[0] == "BLOB" && ids_[1] == "STREAM") + { + r.type = sql_type::BLOB; + r.stream = true; + } + // // Apply the first four rules of the SQLite type to affinity // conversion algorithm. // - if (find ("INT")) + else if (find ("INT")) r.type = sql_type::INTEGER; else if (find ("TEXT") || find ("CHAR") || find ("CLOB")) r.type = sql_type::TEXT; diff --git a/odb/relational/sqlite/context.hxx b/odb/relational/sqlite/context.hxx index 7c5107e..6fcdc47 100644 --- a/odb/relational/sqlite/context.hxx +++ b/odb/relational/sqlite/context.hxx @@ -26,9 +26,10 @@ namespace relational invalid }; - sql_type (): type (invalid) {} + sql_type (): type (invalid), stream (false) {} core_type type; + bool stream; // TEXT or BLOB via sqlite3_blob_open(). // Conversion expressions for custom database types. // diff --git a/odb/relational/sqlite/header.cxx b/odb/relational/sqlite/header.cxx index 9432825..e548413 100644 --- a/odb/relational/sqlite/header.cxx +++ b/odb/relational/sqlite/header.cxx @@ -48,6 +48,15 @@ namespace relational << "bool " << mi.var << "null;" << endl; } + + virtual void + traverse_stream (member_info& mi) + { + os << image_type << " " << mi.var << "value;" + << "std::size_t " << mi.var << "size;" + << "bool " << mi.var << "null;" + << endl; + } }; entry image_member_; } diff --git a/odb/relational/sqlite/model.cxx b/odb/relational/sqlite/model.cxx index 47eb88b..45216b7 100644 --- a/odb/relational/sqlite/model.cxx +++ b/odb/relational/sqlite/model.cxx @@ -23,6 +23,27 @@ namespace relational { object_columns (base const& x): base (x) {} + virtual string + type (semantics::data_member& m) + { + // Translate BLOB|TEXT STREAM to just BLOB|TEXT. + // + string r (relational::object_columns::type (m)); + + sql_type const& t (parse_sql_type (r, m, false)); + if (t.stream) + { + switch (t.type) + { + case sql_type::BLOB: r = "BLOB"; break; + case sql_type::TEXT: r = "TEXT"; break; + default: break; + } + } + + return r; + } + virtual bool null (semantics::data_member& m) { diff --git a/odb/relational/sqlite/source.cxx b/odb/relational/sqlite/source.cxx index 0d821ac..2f6623e 100644 --- a/odb/relational/sqlite/source.cxx +++ b/odb/relational/sqlite/source.cxx @@ -17,6 +17,72 @@ namespace relational { namespace relational = relational::source; + struct query_parameters: relational::query_parameters, context + { + query_parameters (base const& x): base (x) {} + + virtual string + next (semantics::data_member& m, + const string& column, + const string& sqlt) + { + // Handle stream columns. Specifically, we somehow need to + // pass the column name to the code that runs in the + // statement. So what we are going to do is encode it + // in the parameter name. + // + if (sk_ == statement_insert || sk_ == statement_update) + { + const sql_type& t (parse_sql_type (sqlt, m, false)); + if (t.stream) + { + // The column name is quoted. + // + string r (column); + r[0] = '$'; // Replace leading '"'. + r.resize (r.size () - 1); // Remove trailing '"'. + + // Verify it only contains allowed characters. + // + for (size_t i (1); i != r.size (); ++i) + { + char c (r[i]); + if (c != '_' && + (c < '0' || c > '9') && + (c < 'a' || c > 'z') && + (c < 'A' || c > 'Z')) + { + cerr << m.file () << ":" << m.line () << ":" << m.column () + << ": error: unsupported character '" << c << "' in " + << sqlt << " column name " << column << endl; + + cerr << m.file () << ":" << m.line () << ":" << m.column () + << ": info: STREAM column can contain alpha-numeric " + << "characters plus '_'" << endl; + + throw operation_failed (); + } + } + + // For TEXT columns, since we use the *_bind_zeroblob() + // function (there is no *_bind_zerotext()), the value + // that will be stored is BLOB, not TEXT, unless we + // explicitly CAST it. The user better make sure the + // encoding of raw TEXT data they are going to write + // matches the database encoding. + // + if (t.type == sql_type::TEXT) + r = "CAST(" + r + " AS TEXT)"; + + return r; + } + } + + return "?"; + } + }; + entry query_parameters_; + // // bind // @@ -71,6 +137,15 @@ namespace relational "value.capacity ();" << b << ".is_null = &" << arg << "." << mi.var << "null;"; } + + virtual void + traverse_stream (member_info& mi) + { + os << b << ".type = sqlite::bind::stream;" + << b << ".buffer = &" << arg << "." << mi.var << "value;" + << b << ".size = &" << arg << "." << mi.var << "size;" + << b << ".is_null = &" << arg << "." << mi.var << "null;"; + } }; entry bind_member_; @@ -110,6 +185,13 @@ namespace relational << "grew = true;" << "}"; } + + virtual void + traverse_stream (member_info&) + { + os << e << " = false;" + << endl; + } }; entry grow_member_; @@ -166,6 +248,17 @@ namespace relational << "i." << mi.var << "null = is_null;" << "grew = grew || (cap != i." << mi.var << "value.capacity ());"; } + + virtual void + traverse_stream (member_info& mi) + { + os << traits << "::set_image (" << endl + << "i." << mi.var << "value," << endl + << "i." << mi.var << "size," << endl + << "is_null," << endl + << member << ");" + << "i." << mi.var << "null = is_null;"; + } }; entry init_image_member_; @@ -220,10 +313,65 @@ namespace relational << "i." << mi.var << "null);" << endl; } + + virtual void + traverse_stream (member_info& mi) + { + os << traits << "::set_value (" << endl + << member << "," << endl + << "i." << mi.var << "value," << endl + << "i." << mi.var << "size," << endl + << "i." << mi.var << "null);" + << endl; + } }; entry init_value_member_; - struct container_traits: relational::container_traits, context + struct statement_columns_common: context + { + void + process (relational::statement_columns& cs, statement_kind sk) + { + using relational::statement_columns; + + // For SELECT statements, add _ROWID_ "follow-up" column to + // each stream column. The reason we need both, and not just + // ROWID is the NULL value. Let's hope that SELECT'ing a BLOB + // but not actually reading it with sqlite3_result_blob() is + // as fast as not SELECT'ing it. + // + if (sk != statement_select) + return; + + for (statement_columns::iterator i (cs.begin ()); + i != cs.end (); ++i) + { + if (parse_sql_type (i->type, *i->member).stream) + { + // Column is already table-qualified and quoted. Do some + // surgery to replace it with _ROWID_. That is, we want to + // transform "table"."column" to "table"."_ROWID_". + // + string c (i->column); + string::size_type n (c.size ()), p (c.rfind ('"', n - 2)); + assert (p != string::npos); + string as (c, p + 1, n - p - 2); + c.resize (p); + c += "\"_ROWID_\""; + + // We are going to pack this "tightly", without any newlines, + // so that the statement processing code treats them as a + // single column. + // + i->column += ','; + i->column += c; + } + } + } + }; + + struct container_traits: relational::container_traits, + statement_columns_common { container_traits (base const& x): base (x) {} @@ -234,10 +382,32 @@ namespace relational // interleaving statements. // } + + virtual void + process_statement_columns (relational::statement_columns& cols, + statement_kind sk, + bool) + { + statement_columns_common::process (cols, sk); + } }; entry container_traits_; - struct class_: relational::class_, context + struct section_traits: relational::section_traits, + statement_columns_common + { + section_traits (base const& x): base (x) {} + + virtual void + process_statement_columns (relational::statement_columns& cols, + statement_kind sk) + { + statement_columns_common::process (cols, sk); + } + }; + entry section_traits_; + + struct class_: relational::class_, statement_columns_common { class_ (base const& x): base (x) {} @@ -274,6 +444,14 @@ namespace relational return base::join_syntax (vo); } + + virtual void + process_statement_columns (relational::statement_columns& cols, + statement_kind sk, + bool) + { + statement_columns_common::process (cols, sk); + } }; entry class_entry_; } -- cgit v1.1