From e85b07722107d00e4a3182ff4d33274a617bb55a Mon Sep 17 00:00:00 2001 From: Boris Kolpackov Date: Wed, 19 Nov 2014 11:51:17 +0200 Subject: Implement bulk API code generation --- odb/pragma.cxx | 47 ++++- odb/relational/context.cxx | 1 + odb/relational/context.hxx | 1 + odb/relational/header.cxx | 25 +++ odb/relational/mssql/context.cxx | 1 + odb/relational/mssql/header.cxx | 12 +- odb/relational/mysql/context.cxx | 1 + odb/relational/oracle/context.cxx | 1 + odb/relational/oracle/header.cxx | 12 +- odb/relational/pgsql/context.cxx | 1 + odb/relational/processor.cxx | 6 + odb/relational/source.cxx | 360 ++++++++++++++++++++++++++++++++++++++ odb/relational/sqlite/context.cxx | 1 + odb/relational/validator.cxx | 60 +++++++ 14 files changed, 522 insertions(+), 7 deletions(-) diff --git a/odb/pragma.cxx b/odb/pragma.cxx index f40258f..4d0acae 100644 --- a/odb/pragma.cxx +++ b/odb/pragma.cxx @@ -378,7 +378,7 @@ check_spec_decl_type (declaration const& d, if (tc != RECORD_TYPE) { error (l) << "name '" << name << "' in db pragma " << p << " does " - << "not refer to a data member or class" << endl; + << "not refer to a class" << endl; return false; } } @@ -458,7 +458,8 @@ check_spec_decl_type (declaration const& d, p == "optimistic" || p == "polymorphic" || p == "definition" || - p == "sectionable") + p == "sectionable" || + p == "bulk") { if (tc != RECORD_TYPE) { @@ -1479,6 +1480,48 @@ handle_pragma (cxx_lexer& l, tt = l.next (tl, &tn); } + else if (p == "bulk") + { + // bulk (batch-size) + // + + // Make sure we've got the correct declaration type. + // + if (decl && !check_spec_decl_type (decl, decl_name, p, loc)) + return; + + if (l.next (tl, &tn) != CPP_OPEN_PAREN) + { + error (l) << "'(' expected after db pragma " << p << endl; + return; + } + + // base + // + if (l.next (tl, &tn) != CPP_NUMBER || TREE_CODE (tn) != INTEGER_CST) + { + error (l) << "unsigned integer expected as batch size" << endl; + return; + } + + unsigned long long b (integer (tn)); + + if (b == 0 || b == 1) + { + error (l) << "batch size has to be greater than 1" << endl; + return; + } + + val = b; + + if (l.next (tl, &tn) != CPP_CLOSE_PAREN) + { + error (l) << "')' expected at the end of db pragma " << p << endl; + return; + } + + tt = l.next (tl, &tn); + } else if (p == "query") { // query () diff --git a/odb/relational/context.cxx b/odb/relational/context.cxx index b525802..3e5596e 100644 --- a/odb/relational/context.cxx +++ b/odb/relational/context.cxx @@ -28,6 +28,7 @@ namespace relational insert_send_auto_id (current ().insert_send_auto_id), delay_freeing_statement_result (current ().delay_freeing_statement_result), need_image_clone (current ().need_image_clone), + generate_bulk (current ().generate_bulk), global_index (current ().global_index), global_fkey (current ().global_fkey), bind_vector (data_->bind_vector_), diff --git a/odb/relational/context.hxx b/odb/relational/context.hxx index d0211c6..b2a4032 100644 --- a/odb/relational/context.hxx +++ b/odb/relational/context.hxx @@ -270,6 +270,7 @@ namespace relational bool insert_send_auto_id; bool delay_freeing_statement_result; bool need_image_clone; + bool generate_bulk; bool global_index; bool global_fkey; diff --git a/odb/relational/header.cxx b/odb/relational/header.cxx index 41fdd26..6584b8e 100644 --- a/odb/relational/header.cxx +++ b/odb/relational/header.cxx @@ -494,6 +494,12 @@ traverse_object (type& c) os << ");" << endl; + if (c.count ("bulk-persist")) + os << "static void" << endl + << "persist (database&, " << (auto_id ? "" : "const ") << + "object_type**, std::size_t, multiple_exceptions&);" + << endl; + if (id != 0) { // find (id) @@ -542,6 +548,12 @@ traverse_object (type& c) os << ");" << endl; + + if (c.count ("bulk-update")) + os << "static void" << endl + << "update (database&, const object_type**, std::size_t, " << + "multiple_exceptions&);" + << endl; } // erase () @@ -564,6 +576,19 @@ traverse_object (type& c) os << ");" << endl; + if (c.count ("bulk-erase")) + { + os << "static std::size_t" << endl + << "erase (database&, const id_type**, std::size_t, " << + "multiple_exceptions&);" + << endl; + + os << "static void" << endl + << "erase (database&, const object_type**, std::size_t, " << + "multiple_exceptions&);" + << endl; + } + // Sections. // // We treat all polymorphic sections as (potentially) having something diff --git a/odb/relational/mssql/context.cxx b/odb/relational/mssql/context.cxx index a0037eb..e99cdaf 100644 --- a/odb/relational/mssql/context.cxx +++ b/odb/relational/mssql/context.cxx @@ -89,6 +89,7 @@ namespace relational insert_send_auto_id = false; delay_freeing_statement_result = true; need_image_clone = true; + generate_bulk = true; global_index = false; global_fkey = true; data_->bind_vector_ = "mssql::bind*"; diff --git a/odb/relational/mssql/header.cxx b/odb/relational/mssql/header.cxx index 40b6828..3bc2cbd 100644 --- a/odb/relational/mssql/header.cxx +++ b/odb/relational/mssql/header.cxx @@ -31,10 +31,16 @@ namespace relational if (poly_derived || (abst && !poly)) return; - // batch + // Bulk operations batch size. // - os << "static const std::size_t batch = 1UL;" - << endl; + { + unsigned long long b (c.count ("bulk") + ? c.get ("bulk") + : 1); + + os << "static const std::size_t batch = " << b << "UL;" + << endl; + } // rowvesion // diff --git a/odb/relational/mysql/context.cxx b/odb/relational/mysql/context.cxx index 55dd4f7..4ccb206 100644 --- a/odb/relational/mysql/context.cxx +++ b/odb/relational/mysql/context.cxx @@ -84,6 +84,7 @@ namespace relational insert_send_auto_id = true; delay_freeing_statement_result = false; need_image_clone = false; + generate_bulk = false; global_index = false; global_fkey = true; data_->bind_vector_ = "MYSQL_BIND*"; diff --git a/odb/relational/oracle/context.cxx b/odb/relational/oracle/context.cxx index 98cda52..be97796 100644 --- a/odb/relational/oracle/context.cxx +++ b/odb/relational/oracle/context.cxx @@ -85,6 +85,7 @@ namespace relational insert_send_auto_id = false; delay_freeing_statement_result = false; need_image_clone = true; + generate_bulk = true; global_index = true; global_fkey = true; data_->bind_vector_ = "oracle::bind*"; diff --git a/odb/relational/oracle/header.cxx b/odb/relational/oracle/header.cxx index 1004a4c..055ef7b 100644 --- a/odb/relational/oracle/header.cxx +++ b/odb/relational/oracle/header.cxx @@ -243,8 +243,16 @@ namespace relational if (poly_derived || (abst && !poly)) return; - os << "static const std::size_t batch = 1UL;" - << endl; + // Bulk operations batch size. + // + { + unsigned long long b (c.count ("bulk") + ? c.get ("bulk") + : 1); + + os << "static const std::size_t batch = " << b << "UL;" + << endl; + } } }; entry class1_entry_; diff --git a/odb/relational/pgsql/context.cxx b/odb/relational/pgsql/context.cxx index d09fa2c..9e91e31 100644 --- a/odb/relational/pgsql/context.cxx +++ b/odb/relational/pgsql/context.cxx @@ -84,6 +84,7 @@ namespace relational insert_send_auto_id = false; delay_freeing_statement_result = false; need_image_clone = false; + generate_bulk = false; global_index = true; global_fkey = false; data_->bind_vector_ = "pgsql::bind*"; diff --git a/odb/relational/processor.cxx b/odb/relational/processor.cxx index d32f8e5..b70451a 100644 --- a/odb/relational/processor.cxx +++ b/odb/relational/processor.cxx @@ -1024,6 +1024,12 @@ namespace relational virtual void traverse_object (type& c) { + // Remove the bulk pragma if this database doesn't support bulk + // operations. + // + if (c.count ("bulk") && !generate_bulk) + c.remove ("bulk"); + // Process indexes. Here we need to do two things: resolve member // names to member paths and assign names to unnamed indexes. We // are also going to handle the special container indexes. diff --git a/odb/relational/source.cxx b/odb/relational/source.cxx index cb91411..43d347c 100644 --- a/odb/relational/source.cxx +++ b/odb/relational/source.cxx @@ -1449,6 +1449,173 @@ traverse_object (type& c) os << "}"; + // persist() bulk + // + if (c.count ("bulk-persist")) + { + os << "void " << traits << "::" << endl + << "persist (database& db," << endl + << (auto_id ? "" : "const ") << "object_type** objs," << endl + << "std::size_t n," << endl + << "multiple_exceptions& mex)" + << "{" + << "ODB_POTENTIALLY_UNUSED (db);" + << endl + << "using namespace " << db << ";" + << endl; + + os << db << "::connection& conn (" << endl + << db << "::transaction::current ().connection ());" + << "statements_type& sts (" << endl + << "conn.statement_cache ().find_object ());"; + + if (versioned || + uss.count (user_sections::count_new | + user_sections::count_all | + user_sections::count_versioned_only) != 0) + os << "const schema_version_migration& svm (" << + "sts.version_migration (" << schema_name << "));"; + + os << endl; + + os << "for (std::size_t i (0); i != n; ++i)" + << "{" + << "const object_type& obj (*objs[i]);" + << "callback (db, obj, callback_event::pre_persist);" + //@@ assumption: generate_grow is false + //@@ assumption: insert_send_auto_id is false + << "init (sts.image (i), obj, statement_insert" << + (versioned ? ", svm" : "") << ");" + << "}"; + + //@@ assumption: generate_grow is false + os << "binding& imb (sts.insert_image_binding ());" + << "if (imb.version == 0)" + << "{" + << "bind (imb.bind, sts.image (), statement_insert" << + (versioned ? ", svm" : "") << ");" + << "imb.version++;" + << "}"; + + // Bind id image since that's where the returned id and/or version will + // be extracted. + // + bool bv (opt != 0 && optimistic_insert_bind_version (*opt)); + if (bv || auto_id) + { + os << "binding& idb (sts.id_image_binding ());" + //@@ assumption: generate_grow is false + << "if (idb.version == 0)" + << "{" + << "bind (idb.bind, sts.id_image ());" + << "idb.version++;"; + if (opt != 0) + os << "sts.optimistic_id_image_binding ().version++;"; + os << "}"; + } + + os << "insert_statement& st (sts.persist_statement ());" + << "n = st.execute (n, mex);" // Actual number of rows attempted. + << endl; + + os << "for (std::size_t i (0); i != n; ++i)" + << "{" + << "bool r (st.result (i));" // Sets current in mex. + << endl + << "if (mex[i] != 0)" << endl // Pending exception for this position. + << "continue;" + << endl + << "if (!r)" + << "{" + << "mex.insert (i, object_already_persistent ());" + << "continue;" + << "}" + << "if (mex.fatal ())" << endl // Don't do any extra work. + << "continue;" + << endl + << (auto_id ? "" : "const ") << "object_type& obj (*objs[i]);" + << endl; + + // Extract auto id. + // + if (auto_id) + { + member_access& ma (id->get ("set")); + + if (!ma.synthesized) + os << "// From " << location_string (ma.loc, true) << endl; + + if (ma.placeholder ()) + os << ma.translate ("obj", "id (sts.id_image (i))") << ";" + << endl; + else + { + // If this member is const and we have a synthesized direct access, + // then cast away constness. Otherwise, we assume that the user- + // provided expression handles this. + // + bool cast (ma.direct () && const_type (id->type ())); + if (cast) + os << "const_cast< id_type& > (" << endl; + + os << ma.translate ("obj"); + + if (cast) + os << ")"; + + os << " = id (sts.id_image (i));" + << endl; + } + } + + // Reset sections: loaded, unchanged. + // + for (user_sections::iterator i (uss.begin ()); i != uss.end (); ++i) + { + // Skip special sections. + // + if (i->special == user_section::special_version) + continue; + + data_member& m (*i->member); + + // If the section is soft- added or deleted, check the version. + // + unsigned long long av (added (m)); + unsigned long long dv (deleted (m)); + if (av != 0 || dv != 0) + { + os << "if ("; + + if (av != 0) + os << "svm >= schema_version_migration (" << av << "ULL, true)"; + + if (av != 0 && dv != 0) + os << " &&" << endl; + + if (dv != 0) + os << "svm <= schema_version_migration (" << dv << "ULL, true)"; + + os << ")" << endl; + } + + // Section access is always by reference. + // + member_access& ma (m.get ("get")); + if (!ma.synthesized) + os << "// From " << location_string (ma.loc, true) << endl; + + os << ma.translate ("obj") << ".reset (true, false);" + << endl; + } + + os << "callback (db," << endl + << (auto_id ? "static_cast (obj)," : "obj,") << endl + << "callback_event::post_persist);" + << "}" // for + << "}"; // persist () + } + // update () // if (id != 0 && (!readonly || poly)) @@ -2112,6 +2279,110 @@ traverse_object (type& c) os << "}"; } + // update () bulk + // + if (id != 0 && !readonly && c.count ("bulk-update")) + { + os << "void " << traits << "::" << endl + << "update (database& db," << endl + << "const object_type** objs," << endl + << "std::size_t n," << endl + << "multiple_exceptions& mex)" + << "{" + << "ODB_POTENTIALLY_UNUSED (db);" + << endl + << "using namespace " << db << ";" + << "using " << db << "::update_statement;" + << endl + << db << "::connection& conn (" << endl + << db << "::transaction::current ().connection ());" + << "statements_type& sts (" << endl + << "conn.statement_cache ().find_object ());"; + + if (versioned) + os << "const schema_version_migration& svm (" << + "sts.version_migration (" << schema_name << "));"; + + os << endl + << "for (std::size_t i (0); i != n; ++i)" + << "{" + << "const object_type& obj (*objs[i]);" + << "callback (db, obj, callback_event::pre_update);"; + + if (!id_ma->synthesized) + os << "// From " << location_string (id_ma->loc, true) << endl; + + os << "const id_type& id (" << endl + << id_ma->translate ("obj") << ");"; + + os << "init (sts.id_image (i), id);" + //@@ assumption: generate_grow false + << "init (sts.image (i), obj, statement_update);" + << "}"; + + // Update bindings. + // + os << "binding& idb (sts.id_image_binding ());" + << "binding& imb (sts.update_image_binding ());" + << endl; + + //@@ assumption: generate_grow false + // + os << "bool u (false);" // Avoid incrementing version twice. + << "if (imb.version == 0)" + << "{" + << "bind (imb.bind, sts.image (), statement_update" << + (versioned ? ", svm" : "") << ");" + << "imb.version++;" + << "u = true;" + << "}"; + + //@@ assumption: generate_grow false + // + os << "if (idb.version == 0)" + << "{" + << "bind (idb.bind, sts.id_image ());" + << "idb.version++;"; + if (opt != 0) + os << "sts.optimistic_id_image_binding ().version++;"; + os << endl + << "if (!u)" << endl + << "imb.version++;" + << "}"; + + // We don't need the versioned check here since the statement cannot + // be empty; otherwise it would have been an empty object which is + // illegal. + // + os << "update_statement& st (sts.update_statement ());" + << "n = st.execute (n, mex);"; // Actual number of rows attempted. + + os << endl + << "for (std::size_t i (0); i != n; ++i)" + << "{" + << "unsigned long long r (st.result (i));" // Also sets current in mex. + << endl + << "if (mex[i] != 0)" << endl // Pending exception from result(). + << "continue;" + << endl + << "if (r != 1)" + << "{" + << "mex.insert (i," << endl + //@@ assumption: result_unknown + << "(r == update_statement::result_unknown)," << endl + << "object_not_persistent ());" + << "continue;" + << "}" + << "if (mex.fatal ())" << endl // Don't do any extra work. + << "continue;" + << endl + << "const object_type& obj (*objs[i]);" + << "callback (db, obj, callback_event::post_update);" + << "pointer_cache_traits::update (db, obj);" + << "}" // for + << "}"; // update() + } + // erase (id) // size_t erase_containers (has_a (c, test_straight_container)); @@ -2245,6 +2516,62 @@ traverse_object (type& c) os << "}"; } + // erase (id) bulk + // + if (id != 0 && c.count ("bulk-erase")) + { + os << "std::size_t " << traits << "::" << endl + << "erase (database& db," << endl + << "const id_type** ids," << endl + << "std::size_t n," << endl + << "multiple_exceptions& mex)" + << "{" + << "using namespace " << db << ";" + << endl + << "ODB_POTENTIALLY_UNUSED (db);" + << endl + << db << "::connection& conn (" << endl + << db << "::transaction::current ().connection ());" + << "statements_type& sts (" << endl + << "conn.statement_cache ().find_object ());" + << endl + << "for (std::size_t i (0); i != n; ++i)" << endl + << "init (sts.id_image (i), *ids[i]);" + << endl + << "binding& idb (sts.id_image_binding ());" + //@@ assumption: generate_grow false + << "if (idb.version == 0)" + << "{" + << "bind (idb.bind, sts.id_image ());" + << "idb.version++;" + << (opt != 0 ? "sts.optimistic_id_image_binding ().version++;" : "") + << "}" + << "delete_statement& st (sts.erase_statement ());" + << "n = st.execute (n, mex);" // Set to actual number of rows attempted. + << endl + << "for (std::size_t i (0); i != n; ++i)" + << "{" + << "unsigned long long r (st.result (i));" // Sets current in mex. + << endl + << "if (mex[i] != 0)" << endl // Pending exception from result(). + << "continue;" + << endl + << "if (r != 1)" + << "{" + << "mex.insert (i," << endl + //@@ assumption: result_unknown + << "(r == delete_statement::result_unknown)," << endl + << "object_not_persistent ());" + << "continue;" + << "}" + << "if (mex.fatal ())" << endl // Don't do any extra work. + << "continue;" + << "pointer_cache_traits::erase (db, *ids[i]);" + << "}" // for + << "return n;" + << "}"; // erase() + } + // erase (object) // bool erase_smart_containers (has_a (c, test_smart_container)); @@ -2557,6 +2884,39 @@ traverse_object (type& c) os << "}"; } + // erase (object) bulk + // + if (id != 0 && c.count ("bulk-erase")) + { + os << "void " << traits << "::" << endl + << "erase (database& db," << endl + << "const object_type** objs," << endl + << "std::size_t n," << endl + << "multiple_exceptions& mex)" + << "{" + << "id_type a[batch];" + << "const id_type* p[batch];" + << endl + << "for (std::size_t i (0); i != n; ++i)" + << "{" + << "const object_type& obj (*objs[i]);" + << "callback (db, obj, callback_event::pre_erase);" + << "a[i] = id (obj);" + << "p[i] = a + i;" + << "}" + << "n = erase (db, p, n, mex);" + << endl + << "if (mex.fatal ())" << endl // Don't do any extra work. + << "return;" + << endl + << "for (std::size_t i (0); i != n; ++i)" + << "{" + << "if (mex[i] == 0)" << endl // No pending exception. + << "callback (db, *objs[i], callback_event::post_erase);" + << "}" // for + << "}"; // erase() + } + // find (id) // if (id != 0 && c.default_ctor ()) diff --git a/odb/relational/sqlite/context.cxx b/odb/relational/sqlite/context.cxx index f347d30..76c2b02 100644 --- a/odb/relational/sqlite/context.cxx +++ b/odb/relational/sqlite/context.cxx @@ -86,6 +86,7 @@ namespace relational insert_send_auto_id = true; delay_freeing_statement_result = false; need_image_clone = false; + generate_bulk = false; global_index = true; global_fkey = false; data_->bind_vector_ = "sqlite::bind*"; diff --git a/odb/relational/validator.cxx b/odb/relational/validator.cxx index 9705121..f204a6d 100644 --- a/odb/relational/validator.cxx +++ b/odb/relational/validator.cxx @@ -387,6 +387,66 @@ namespace relational names (c, data_member_names_); + // Validate bulk operation support. + // + for (bool i (true); i && c.count ("bulk"); i = false) + { + location_t l (c.get ("bulk-location")); + + if (polymorphic (c)) + { + error (l) << "bulk operations on polymorphic objects are " + "not supported" << endl; + valid_ = false; + break; + } + + if (has_a (c, test_straight_container)) + { + error (l) << "bulk operations on objects with containers are " + "not supported" << endl; + valid_ = false; + break; + } + + if (optimistic (c)) + { + error (l) << "bulk operations on optimistic objects are not " + "supported" << endl; + valid_ = false; + break; + } + + bool update (true); + + // If we have a change-updated section, then we cannot generate + // the bulk update operation. + // + user_sections& uss (c.get ("user-sections")); + + for (user_sections::iterator i (uss.begin ()); + update && i != uss.end (); + ++i) + { + const user_section& s (*i); + + // Skip special sections. + // + if (s.special != user_section::special_ordinary) + continue; + + // Always-updated section still needs a separate statement + // (since it may not be loaded). + // + if (!s.update_empty () && s.update != user_section::update_manual) + update = false; + } + + c.set ("bulk-persist", true); + if (update) c.set ("bulk-update", true); + c.set ("bulk-erase", true); + } + // Validate indexes. // { -- cgit v1.1