From 8f6a9c51bc64226d7c296e4b0172f9e56a7eea3b Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Wed, 15 Jul 2015 18:43:03 +0200 Subject: Implement SQLite incremental BLOB/TEXT I/O --- odb/relational/sqlite/common.cxx | 28 +++++- odb/relational/sqlite/common.hxx | 26 ++++++ odb/relational/sqlite/context.cxx | 15 +++- odb/relational/sqlite/context.hxx | 3 +- odb/relational/sqlite/header.cxx | 9 ++ odb/relational/sqlite/model.cxx | 21 +++++ odb/relational/sqlite/source.cxx | 182 +++++++++++++++++++++++++++++++++++++- 7 files changed, 278 insertions(+), 6 deletions(-) (limited to 'odb/relational/sqlite') diff --git a/odb/relational/sqlite/common.cxx b/odb/relational/sqlite/common.cxx index 36a0e86..4ab8968 100644 --- a/odb/relational/sqlite/common.cxx +++ b/odb/relational/sqlite/common.cxx @@ -39,12 +39,18 @@ namespace relational } case sql_type::TEXT: { - traverse_text (mi); + if (mi.st->stream) + traverse_text_stream (mi); + else + traverse_text (mi); break; } case sql_type::BLOB: { - traverse_blob (mi); + if (mi.st->stream) + traverse_blob_stream (mi); + else + traverse_blob (mi); break; } case sql_type::invalid: @@ -108,6 +114,12 @@ namespace relational type_ = "details::buffer"; } + void member_image_type:: + traverse_stream (member_info&) + { + type_ = "sqlite::stream_buffers"; + } + entry member_image_type_; // @@ -170,6 +182,18 @@ namespace relational type_id_ = "sqlite::id_blob"; } + void member_database_type_id:: + traverse_text_stream (member_info&) + { + type_id_ = "sqlite::id_text_stream"; + } + + void member_database_type_id:: + traverse_blob_stream (member_info&) + { + type_id_ = "sqlite::id_blob_stream"; + } + entry member_database_type_id_; // diff --git a/odb/relational/sqlite/common.hxx b/odb/relational/sqlite/common.hxx index bea0889..b5a61a1 100644 --- a/odb/relational/sqlite/common.hxx +++ b/odb/relational/sqlite/common.hxx @@ -56,6 +56,23 @@ namespace relational traverse_string (member_info&) { } + + virtual void + traverse_text_stream (member_info& m) + { + traverse_stream (m); + } + + virtual void + traverse_blob_stream (member_info& m) + { + traverse_stream (m); + } + + virtual void + traverse_stream (member_info&) + { + } }; struct member_image_type: relational::member_image_type, @@ -82,6 +99,9 @@ namespace relational virtual void traverse_string (member_info&); + virtual void + traverse_stream (member_info&); + private: string type_; }; @@ -114,6 +134,12 @@ namespace relational virtual void traverse_blob (member_info&); + virtual void + traverse_text_stream (member_info&); + + virtual void + traverse_blob_stream (member_info&); + private: string type_id_; }; diff --git a/odb/relational/sqlite/context.cxx b/odb/relational/sqlite/context.cxx index 007d67f..94b8067 100644 --- a/odb/relational/sqlite/context.cxx +++ b/odb/relational/sqlite/context.cxx @@ -339,10 +339,23 @@ namespace relational if (ids_.empty ()) return error ("expected SQLite type name"); + // First check our own types. + // + if (ids_.size () == 2 && ids_[0] == "TEXT" && ids_[1] == "STREAM") + { + r.type = sql_type::TEXT; + r.stream = true; + } + if (ids_.size () == 2 && ids_[0] == "BLOB" && ids_[1] == "STREAM") + { + r.type = sql_type::BLOB; + r.stream = true; + } + // // Apply the first four rules of the SQLite type to affinity // conversion algorithm. // - if (find ("INT")) + else if (find ("INT")) r.type = sql_type::INTEGER; else if (find ("TEXT") || find ("CHAR") || find ("CLOB")) r.type = sql_type::TEXT; diff --git a/odb/relational/sqlite/context.hxx b/odb/relational/sqlite/context.hxx index 7c5107e..6fcdc47 100644 --- a/odb/relational/sqlite/context.hxx +++ b/odb/relational/sqlite/context.hxx @@ -26,9 +26,10 @@ namespace relational invalid }; - sql_type (): type (invalid) {} + sql_type (): type (invalid), stream (false) {} core_type type; + bool stream; // TEXT or BLOB via sqlite3_blob_open(). // Conversion expressions for custom database types. // diff --git a/odb/relational/sqlite/header.cxx b/odb/relational/sqlite/header.cxx index 9432825..e548413 100644 --- a/odb/relational/sqlite/header.cxx +++ b/odb/relational/sqlite/header.cxx @@ -48,6 +48,15 @@ namespace relational << "bool " << mi.var << "null;" << endl; } + + virtual void + traverse_stream (member_info& mi) + { + os << image_type << " " << mi.var << "value;" + << "std::size_t " << mi.var << "size;" + << "bool " << mi.var << "null;" + << endl; + } }; entry image_member_; } diff --git a/odb/relational/sqlite/model.cxx b/odb/relational/sqlite/model.cxx index 47eb88b..45216b7 100644 --- a/odb/relational/sqlite/model.cxx +++ b/odb/relational/sqlite/model.cxx @@ -23,6 +23,27 @@ namespace relational { object_columns (base const& x): base (x) {} + virtual string + type (semantics::data_member& m) + { + // Translate BLOB|TEXT STREAM to just BLOB|TEXT. + // + string r (relational::object_columns::type (m)); + + sql_type const& t (parse_sql_type (r, m, false)); + if (t.stream) + { + switch (t.type) + { + case sql_type::BLOB: r = "BLOB"; break; + case sql_type::TEXT: r = "TEXT"; break; + default: break; + } + } + + return r; + } + virtual bool null (semantics::data_member& m) { diff --git a/odb/relational/sqlite/source.cxx b/odb/relational/sqlite/source.cxx index 0d821ac..2f6623e 100644 --- a/odb/relational/sqlite/source.cxx +++ b/odb/relational/sqlite/source.cxx @@ -17,6 +17,72 @@ namespace relational { namespace relational = relational::source; + struct query_parameters: relational::query_parameters, context + { + query_parameters (base const& x): base (x) {} + + virtual string + next (semantics::data_member& m, + const string& column, + const string& sqlt) + { + // Handle stream columns. Specifically, we somehow need to + // pass the column name to the code that runs in the + // statement. So what we are going to do is encode it + // in the parameter name. + // + if (sk_ == statement_insert || sk_ == statement_update) + { + const sql_type& t (parse_sql_type (sqlt, m, false)); + if (t.stream) + { + // The column name is quoted. + // + string r (column); + r[0] = '$'; // Replace leading '"'. + r.resize (r.size () - 1); // Remove trailing '"'. + + // Verify it only contains allowed characters. + // + for (size_t i (1); i != r.size (); ++i) + { + char c (r[i]); + if (c != '_' && + (c < '0' || c > '9') && + (c < 'a' || c > 'z') && + (c < 'A' || c > 'Z')) + { + cerr << m.file () << ":" << m.line () << ":" << m.column () + << ": error: unsupported character '" << c << "' in " + << sqlt << " column name " << column << endl; + + cerr << m.file () << ":" << m.line () << ":" << m.column () + << ": info: STREAM column can contain alpha-numeric " + << "characters plus '_'" << endl; + + throw operation_failed (); + } + } + + // For TEXT columns, since we use the *_bind_zeroblob() + // function (there is no *_bind_zerotext()), the value + // that will be stored is BLOB, not TEXT, unless we + // explicitly CAST it. The user better make sure the + // encoding of raw TEXT data they are going to write + // matches the database encoding. + // + if (t.type == sql_type::TEXT) + r = "CAST(" + r + " AS TEXT)"; + + return r; + } + } + + return "?"; + } + }; + entry query_parameters_; + // // bind // @@ -71,6 +137,15 @@ namespace relational "value.capacity ();" << b << ".is_null = &" << arg << "." << mi.var << "null;"; } + + virtual void + traverse_stream (member_info& mi) + { + os << b << ".type = sqlite::bind::stream;" + << b << ".buffer = &" << arg << "." << mi.var << "value;" + << b << ".size = &" << arg << "." << mi.var << "size;" + << b << ".is_null = &" << arg << "." << mi.var << "null;"; + } }; entry bind_member_; @@ -110,6 +185,13 @@ namespace relational << "grew = true;" << "}"; } + + virtual void + traverse_stream (member_info&) + { + os << e << " = false;" + << endl; + } }; entry grow_member_; @@ -166,6 +248,17 @@ namespace relational << "i." << mi.var << "null = is_null;" << "grew = grew || (cap != i." << mi.var << "value.capacity ());"; } + + virtual void + traverse_stream (member_info& mi) + { + os << traits << "::set_image (" << endl + << "i." << mi.var << "value," << endl + << "i." << mi.var << "size," << endl + << "is_null," << endl + << member << ");" + << "i." << mi.var << "null = is_null;"; + } }; entry init_image_member_; @@ -220,10 +313,65 @@ namespace relational << "i." << mi.var << "null);" << endl; } + + virtual void + traverse_stream (member_info& mi) + { + os << traits << "::set_value (" << endl + << member << "," << endl + << "i." << mi.var << "value," << endl + << "i." << mi.var << "size," << endl + << "i." << mi.var << "null);" + << endl; + } }; entry init_value_member_; - struct container_traits: relational::container_traits, context + struct statement_columns_common: context + { + void + process (relational::statement_columns& cs, statement_kind sk) + { + using relational::statement_columns; + + // For SELECT statements, add _ROWID_ "follow-up" column to + // each stream column. The reason we need both, and not just + // ROWID is the NULL value. Let's hope that SELECT'ing a BLOB + // but not actually reading it with sqlite3_result_blob() is + // as fast as not SELECT'ing it. + // + if (sk != statement_select) + return; + + for (statement_columns::iterator i (cs.begin ()); + i != cs.end (); ++i) + { + if (parse_sql_type (i->type, *i->member).stream) + { + // Column is already table-qualified and quoted. Do some + // surgery to replace it with _ROWID_. That is, we want to + // transform "table"."column" to "table"."_ROWID_". + // + string c (i->column); + string::size_type n (c.size ()), p (c.rfind ('"', n - 2)); + assert (p != string::npos); + string as (c, p + 1, n - p - 2); + c.resize (p); + c += "\"_ROWID_\""; + + // We are going to pack this "tightly", without any newlines, + // so that the statement processing code treats them as a + // single column. + // + i->column += ','; + i->column += c; + } + } + } + }; + + struct container_traits: relational::container_traits, + statement_columns_common { container_traits (base const& x): base (x) {} @@ -234,10 +382,32 @@ namespace relational // interleaving statements. // } + + virtual void + process_statement_columns (relational::statement_columns& cols, + statement_kind sk, + bool) + { + statement_columns_common::process (cols, sk); + } }; entry container_traits_; - struct class_: relational::class_, context + struct section_traits: relational::section_traits, + statement_columns_common + { + section_traits (base const& x): base (x) {} + + virtual void + process_statement_columns (relational::statement_columns& cols, + statement_kind sk) + { + statement_columns_common::process (cols, sk); + } + }; + entry section_traits_; + + struct class_: relational::class_, statement_columns_common { class_ (base const& x): base (x) {} @@ -274,6 +444,14 @@ namespace relational return base::join_syntax (vo); } + + virtual void + process_statement_columns (relational::statement_columns& cols, + statement_kind sk, + bool) + { + statement_columns_common::process (cols, sk); + } }; entry class_entry_; } -- cgit v1.1