- developers - lists.mariadb.org

Re: [Maria-developers] 19bdb03e97d: MDEV-21211 LOCK_plugin optimization.
by Sergei Golubchik 02 Oct '20

02 Oct '20

Hi, Alexey! Thanks, this is great! Comments are below, there aren't many. The main concern - you only do RCU when installingplugins, but not when uninstalling. I'm not sure how it's supposed to work. On Oct 02, Alexey Botchkov wrote: > revision-id: 19bdb03e97d (mariadb-10.4.11-337-g19bdb03e97d) > parent(s): 0041dacc1b8 > author: Alexey Botchkov <holyfoot(a)mariadb.com> > committer: Alexey Botchkov <holyfoot(a)mariadb.com> > timestamp: 2020-08-11 00:01:53 +0400 > message: > > MDEV-21211 LOCK_plugin optimization. > > Use Apc_target calls to 'notify' threads about changes in plugins. > diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc > index bf33148811e..1f38c55b922 100644 > --- a/sql/sql_parse.cc > +++ b/sql/sql_parse.cc > @@ -1348,10 +1348,10 @@ bool do_command(THD *thd) > my_net_set_read_timeout(net, thd->variables.net_read_timeout); > > DBUG_ASSERT(packet_length); > - DBUG_ASSERT(!thd->apc_target.is_enabled()); > + thd->apc_target.enable(); > return_value= dispatch_command(command, thd, packet+1, > (uint) (packet_length-1), FALSE, FALSE); > - DBUG_ASSERT(!thd->apc_target.is_enabled()); > + thd->apc_target.disable(); is this because you don't want to post apc requests to threads that wait for the client? > > out: > thd->lex->restore_set_statement_var(); > diff --git a/sql/sql_plugin.cc b/sql/sql_plugin.cc > index 981420f03ae..84438e3ff70 100644 > --- a/sql/sql_plugin.cc > +++ b/sql/sql_plugin.cc > @@ -217,17 +217,28 @@ static struct > #include "sql_plugin_services.ic" > > /* > - A mutex LOCK_plugin must be acquired before accessing the > - following variables/structures. > - We are always manipulating ref count, so a rwlock here is unneccessary. > + A mutex LOCK_plugin must be acquired when we change the > + new_plugins_state content and when we swap the global_plugins_state > + and the new_plugins_state. > */ > mysql_mutex_t LOCK_plugin; > +struct st_plugins_state > +{ > + DYNAMIC_ARRAY m_array; > + HASH m_hash[MYSQL_MAX_PLUGIN_TYPE_NUM]; > +#ifndef DBUG_OOF DBUG_OFF > + uint m_ref_counter; > +#endif > +}; > + > static DYNAMIC_ARRAY plugin_dl_array; > -static DYNAMIC_ARRAY plugin_array; > -static HASH plugin_hash[MYSQL_MAX_PLUGIN_TYPE_NUM]; > static MEM_ROOT plugin_mem_root; > static bool reap_needed= false; > volatile int global_plugin_version= 1; > +static st_plugins_state plugins_states[2]; > +st_plugins_state *global_plugins_state= plugins_states; it seems this can be static too, you don't use it outside of sql_plugin.cc (and making it global_plugins_state= &plugins_states[0], new_plugins_state= &plugins_state[1]; would be a bit more explicit) > +static st_plugins_state *new_plugins_state= plugins_states + 1; > + > > static bool initialized= 0; > ulong dlopen_count; > @@ -871,7 +882,8 @@ static void plugin_dl_del(struct st_plugin_dl *plugin_dl) > } > > > -static struct st_plugin_int *plugin_find_internal(const LEX_CSTRING *name, > +static struct st_plugin_int *plugin_find_internal(st_plugins_state *st, shouldn't `st` be const here too? > + const LEX_CSTRING *name, > int type) > { > uint i; > @@ -879,21 +891,19 @@ static struct st_plugin_int *plugin_find_internal(const LEX_CSTRING *name, > if (! initialized) > DBUG_RETURN(0); > > - mysql_mutex_assert_owner(&LOCK_plugin); may be you'd want here something like if (st == new_plugins_state) mysql_mutex_assert_owner(&LOCK_plugin); > - > if (type == MYSQL_ANY_PLUGIN) > { > for (i= 0; i < MYSQL_MAX_PLUGIN_TYPE_NUM; i++) > { > struct st_plugin_int *plugin= (st_plugin_int *) > - my_hash_search(&plugin_hash[i], (const uchar *)name->str, name->length); > + my_hash_search(&st->m_hash[i], (const uchar *)name->str, name->length); > if (plugin) > DBUG_RETURN(plugin); > } > } > else > DBUG_RETURN((st_plugin_int *) > - my_hash_search(&plugin_hash[type], (const uchar *)name->str, > + my_hash_search(&st->m_hash[type], (const uchar *)name->str, > name->length)); > DBUG_RETURN(0); > } > @@ -1210,13 +1209,6 @@ static void plugin_variables_deinit(struct st_plugin_int *plugin) > > static void plugin_deinitialize(struct st_plugin_int *plugin, bool ref_check) > { > - /* > - we don't want to hold the LOCK_plugin mutex as it may cause > - deinitialization to deadlock if plugins have worker threads > - with plugin locks > - */ > - mysql_mutex_assert_not_owner(&LOCK_plugin); does it mean you *do* keep LOCK_plugin here now? > - > if (plugin->plugin->status_vars) > { > /* > @@ -1280,24 +1273,25 @@ static void plugin_del(struct st_plugin_int *plugin) > DBUG_VOID_RETURN; > } > > + > static void reap_plugins(void) > { > uint count; > struct st_plugin_int *plugin, **reap, **list; > > - mysql_mutex_assert_owner(&LOCK_plugin); > - > if (!reap_needed) > return; > > + mysql_mutex_lock(&LOCK_plugin); > reap_needed= false; > - count= plugin_array.elements; > + count= global_plugins_state->m_array.elements; > reap= (struct st_plugin_int **)my_alloca(sizeof(plugin)*(count+1)); > *(reap++)= NULL; > > for (uint i=0; i < MYSQL_MAX_PLUGIN_TYPE_NUM; i++) > { > - HASH *hash= plugin_hash + plugin_type_initialization_order[i]; > + HASH *hash= global_plugins_state->m_hash + > + plugin_type_initialization_order[i]; do you need LOCK_plugin when changing plugin states? your comment at the LOCK_plugin declaration doesn't mention it. > for (uint j= 0; j < hash->records; j++) > { > plugin= (struct st_plugin_int *) my_hash_element(hash, j); > @@ -1542,6 +1529,38 @@ static void init_plugin_psi_keys(void) > } > #endif /* HAVE_PSI_INTERFACE */ > > + > +static bool copy_plugins_state(st_plugins_state *dest, st_plugins_state *src) > +{ > + uint i; > + mysql_mutex_assert_owner ? for documentation purposes. > + reset_dynamic(&dest->m_array); > + for (i= 0; i < MYSQL_MAX_PLUGIN_TYPE_NUM; i++) > + my_hash_reset(&dest->m_hash[i]); > + > + for (i= 0; i < src->m_array.elements; i++) > + { > + struct st_plugin_int *ptr; > + ptr= *dynamic_element(&src->m_array, i, struct st_plugin_int **); > + if (insert_dynamic(&dest->m_array, (uchar *) &ptr) || > + (ptr->state != PLUGIN_IS_FREED && > + my_hash_insert(&dest->m_hash[ptr->plugin->type], > + (uchar*) ptr))) > + goto err; > + } > + > + return false; > +err: > + return true; > +} > + > + > +static void set_new_global_plugins_state() > +{ mysql_mutex_assert_owner ? for documentation purposes. > + swap_variables(st_plugins_state *, global_plugins_state, new_plugins_state); > +} > + > + > /* > The logic is that we first load and initialize all compiled in plugins. > From there we load up the dynamic types (assuming we have not been told to > @@ -1815,7 +1834,9 @@ static void plugin_load(MEM_ROOT *tmp_root) > tables.init_one_table(&MYSQL_SCHEMA_NAME, &MYSQL_PLUGIN_NAME, 0, TL_READ); > tables.open_strategy= TABLE_LIST::OPEN_NORMAL; > > + set_new_global_plugins_state(); > result= open_and_lock_tables(new_thd, &tables, FALSE, MYSQL_LOCK_IGNORE_TIMEOUT); > + set_new_global_plugins_state(); why here, why twice? > > table= tables.table; > if (result) > @@ -1851,52 +1872,31 @@ static void plugin_load(MEM_ROOT *tmp_root) > if (!name.length || !dl.length) > continue; > > - /* > - Pre-acquire audit plugins for events that may potentially occur > - during [UN]INSTALL PLUGIN. > - > - When audit event is triggered, audit subsystem acquires interested > - plugins by walking through plugin list. Evidently plugin list > - iterator protects plugin list by acquiring LOCK_plugin, see > - plugin_foreach_with_mask(). > - > - On the other hand plugin_load is acquiring LOCK_plugin > - rather for a long time. > - > - When audit event is triggered during plugin_load plugin > - list iterator acquires the same lock (within the same thread) > - second time. > - > - This hack should be removed when LOCK_plugin is fixed so it > - protects only what it supposed to protect. > - > - See also mysql_install_plugin(), mysql_uninstall_plugin() and > - initialize_audit_plugin() > - */ > - if (mysql_audit_general_enabled()) > - mysql_audit_acquire_plugins(new_thd, event_class_mask); > - \o/ > /* > there're no other threads running yet, so we don't need a mutex. > but plugin_add() before is designed to work in multi-threaded > environment, and it uses mysql_mutex_assert_owner(), so we lock > the mutex here to satisfy the assert > */ > - mysql_mutex_lock(&LOCK_plugin); > plugin_add(tmp_root, false, &name, &dl, MYF(ME_ERROR_LOG)); > free_root(tmp_root, MYF(MY_MARK_BLOCKS_FREE)); > - mysql_mutex_unlock(&LOCK_plugin); > } > if (unlikely(error > 0)) > sql_print_error(ER_THD(new_thd, ER_GET_ERRNO), my_errno, > table->file->table_type()); > + set_new_global_plugins_state(); > end_read_record(&read_record_info); > table->mark_table_for_reopen(); > close_mysql_tables(new_thd); > -end: > +err_ret: > new_thd->db= null_clex_str; // Avoid free on thd->db > delete new_thd; > + set_new_global_plugins_state(); > DBUG_VOID_RETURN; > + > +end: > + set_new_global_plugins_state(); > + goto err_ret; eh, really? you swap twice again? > } > > > @@ -2058,13 +2055,8 @@ void plugin_shutdown(void) > plugin_deinitialize(plugins[i], false); > } > > - /* > - It's perfectly safe not to lock LOCK_plugin, as there're no > - concurrent threads anymore. But some functions called from here > - use mysql_mutex_assert_owner(), so we lock the mutex to satisfy it > - */ > - mysql_mutex_lock(&LOCK_plugin); > > + mysql_mutex_lock(&LOCK_plugin); the comment was helpful, I think > /* > We defer checking ref_counts until after all plugins are deinitialized > as some may have worker threads holding on to plugin references. > @@ -2176,6 +2173,44 @@ static bool finalize_install(THD *thd, TABLE *table, const LEX_CSTRING *name, > return 0; > } > > + > +/* > + Request object for ending the new_plugins_state change. > +*/ > + > +class Ping_all_threads_request : public Apc_target::Apc_call > +{ > +public: > + /* Overloaded virtual functions. */ > + void call_in_target_thread(); > +}; > + > + > +void Ping_all_threads_request::call_in_target_thread() > +{ > + /* Do nothing. We just make sure the thread got to this point. */ > +} > + > + > +static my_bool fix_plugins_state_callback(THD *thd, THD *caller_thd) > +{ > + bool timed_out; > + int timeout_sec= 30; > + > + Ping_all_threads_request ping_req; > + > + mysql_mutex_lock(&thd->LOCK_thd_kill); > + if (thd->get_command() == COM_SLEEP) here you also skip threads waiting on the client? > + { > + mysql_mutex_unlock(&thd->LOCK_thd_kill); > + return FALSE; > + } > + thd->apc_target.make_apc_call(caller_thd, &ping_req, > + timeout_sec, &timed_out); > + return FALSE; > +} > + > + > bool mysql_install_plugin(THD *thd, const LEX_CSTRING *name, > const LEX_CSTRING *dl_arg) > { > @@ -2252,12 +2272,19 @@ bool mysql_install_plugin(THD *thd, const LEX_CSTRING *name, > > if (unlikely(error != INSTALL_GOOD)) > { > + mysql_mutex_unlock(&LOCK_plugin); > reap_needed= true; > reap_plugins(); > + goto err; > } > -err: > global_plugin_version++; > + set_new_global_plugins_state(); > + thd->apc_target.disable(); > mysql_mutex_unlock(&LOCK_plugin); > + server_threads.iterate(fix_plugins_state_callback, thd); > + thd->apc_target.enable(); a comment be good here. to explain why you disable apc and why disable/unlock/iterate must happen in this particlar order. > + > +err: > if (argv) > free_defaults(argv); > DBUG_RETURN(error == INSTALL_FAIL_NOT_OK); > @@ -2414,10 +2416,12 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_CSTRING *name, > error|= !MyFlags; > } > } > - reap_plugins(); > - > global_plugin_version++; > + thd->apc_target.disable(); > mysql_mutex_unlock(&LOCK_plugin); > + server_threads.iterate(fix_plugins_state_callback, thd); > + reap_plugins(); > + thd->apc_target.enable(); I don't quite understand this. You don't do copy-update here. I presume because plugins are only marked PLUGIN_IS_FREED but not actually removed from the array or hash? but you modify plugin state under a mutex, and read it without a mutex. how is that supposed to work? > DBUG_RETURN(error); > #ifdef WITH_WSREP > wsrep_error_label: Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 6f211b1f01b: MDEV-23518 Syntax error in ond SP results in misleading message on SHOW CREATE PROCEDURE
by Sergei Golubchik 02 Oct '20

02 Oct '20

Hi, Oleksandr! On Oct 01, Oleksandr Byelkin wrote: > revision-id: 6f211b1f01b (mariadb-10.4.11-369-g6f211b1f01b) > parent(s): 571764c04fc > author: Oleksandr Byelkin <sanja(a)mariadb.com> > committer: Oleksandr Byelkin <sanja(a)mariadb.com> > timestamp: 2020-08-28 16:05:38 +0200 > message: > > MDEV-23518 Syntax error in ond SP results in misleading message on SHOW CREATE PROCEDURE > > Add info to the error message how to get details about error which happened. > > diff --git a/mysql-test/main/errors.test b/mysql-test/main/errors.test > index f3aedc8e491..a4b95d48e5d 100644 > --- a/mysql-test/main/errors.test > +++ b/mysql-test/main/errors.test > @@ -246,3 +246,39 @@ SELECT UPDATEXML(-73 * -2465717823867977728,@@global.auto_increment_increment,nu > SELECT UPDATEXML(-73 * -2465717823867977728,@@global.long_query_time,null); > > -- echo # End of 10.3 tests > + > +--echo # > +--echo # MDEV-23518: Syntax error in ond SP results in misleading > +--echo # message on SHOW CREATE PROCEDURE > +--echo # > + > +DELIMITER $$; > + > +CREATE PROCEDURE P1 () > +BEGIN NOT ATOMIC > + IF (SELECT 2) THEN > + SELECT 4; > + END IF ; > +END; > +$$ > + > +DELIMITER ;$$ you could've had just CREATE PROCEDURE P1 () SELECT 1; one line, no need to change a delimiter. You overwrite the body below anyway :) > + > +select name,db,body from mysql.proc where name = "P1"; > +update mysql.proc set body_utf8="BEGIN NOT ATOMIC > +IF (SELECT 2) OR foo = 3 THEN > +SELECT 4; > +END IF ; > +END", body="BEGIN NOT ATOMIC > +IF (SELECT 2) OR foo = 3 THEN > +SELECT 4; > +END IF ; > +END"where name = "P1"; > + > +--error ER_SP_PROC_TABLE_CORRUPT > +show create procedure P1; > +show warnings; > + > +drop procedure P1; > + > diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt > index ddc52702e42..582c80e2ad8 100644 > --- a/sql/share/errmsg-utf8.txt > +++ b/sql/share/errmsg-utf8.txt > @@ -5714,8 +5714,7 @@ ER_SP_RECURSION_LIMIT > eng "Recursive limit %d (as set by the max_sp_recursion_depth variable) was exceeded for routine %.192s" > ger "Rekursionsgrenze %d (durch Variable max_sp_recursion_depth gegeben) wurde für Routine %.192s überschritten" > ER_SP_PROC_TABLE_CORRUPT > - eng "Failed to load routine %-.192s. The table mysql.proc is missing, corrupt, or contains bad data (internal code %d)" > - ger "Routine %-.192s konnte nicht geladen werden. Die Tabelle mysql.proc fehlt, ist beschädigt, oder enthält fehlerhaften Daten (interner Code: %d)" > + eng "Failed to load routine %-.192s. The table mysql.proc is missing, corrupt, etc. (internal code %d). For more details, run SHOW WARNINGS" Okay, so you just extended the error message. Perhaps "The table mysql.proc is missing, corrupt, etc." is now redundant and shoud go away too? Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 1

Re: [Maria-developers] New patch on the JSON_TABLE.
by Sergey Petrunia 27 Sep '20

27 Sep '20

Hi Alexey, On Mon, Sep 14, 2020 at 03:38:38PM +0400, Alexey Botchkov wrote: > Hi, Sergei!. > > so the branch name is 'bb-mdef17399-hf'. > It has slightly changed since last patch so please pay attention. > > There i present the patch that resolves issues that you mentioned > in your last comments. > One important is not fixed though. > That is: > CREATE TABLE t1(id INT, f1 JSON); > INSERT INTO t1 VALUES (1, '{\"1\": 1}'), (2, '{\"1\": 2}'), (3, '{\"1\": > 3}'), > (4, '{\"1\": 4}'), (5, '{\"1\": 5}'), (6, '{\"1\": 6}'); > > SELECT * FROM t1 WHERE id IN > (SELECT id FROM t1 as tt2, > JSON_TABLE(f1, "$" COLUMNS (jf FOR ORDINALITY)) AS tbl); > > That SELECT crashes in the optimizer, as the greedy_search() can't find any > satisfying plan. > I got rid of the crash with this line: > @@ -9520,7 +9528,7 @@ best_extension_by_limited_search(JOIN *join, > table_map real_table_bit= s->table->map; > if ((remaining_tables & real_table_bit) && > (allowed_tables & real_table_bit) && > - !(remaining_tables & s->dependent) && > +// !(remaining_tables & s->dependent) && > > But i don't mean it's an acceptable fix. > > It seems to me the problem is in the optimizer, so i'd like to ask your > opinion. > Maybe you just know the answer at once. After some debugging I see that the following happens: 1. Execution reaches Table_function_json_table::setup It computes m_dep_tables= m_json->used_tables(); this is =1 (In the subquery, table tt2 has map=1) 2. Then, subquery is converted into a semi-join. That is, it is merged into its parent subquery. There, we have: table t1: map=1 table tt2: map=2 table JSON_TABLE(...) AS tbl: map=3 3. The subquery was uncorrelated, so SJ-Materialization is a possible option and the join optimizer attempts to construct a join order for the subquery tables. However it can't succeed, as the table "JSON_TABLE(..) AS tbl" is set to be dependent on table with map=1, which is not put into the join order because it is not a part of subquery. As a result, we get assertion failure when the optimizer fails to produce any join orders. The probem here is on step #2. When the subquery's tables are moved to the parent query, they get new values of TABLE::map, and also all their attributes that contain table maps need to be updated. This done here in convert_subq_to_sj() function: ``` /* n. Walk through child's tables and adjust table->map */ List_iterator_fast<TABLE_LIST> si(subq_lex->leaf_tables); while ((tl= si++)) { tl->set_tablenr(table_no); if (tl->is_jtbm()) { tl->jtbm_table_no= table_no; Item *dummy= tl->jtbm_subselect; tl->jtbm_subselect->fix_after_pullout(parent_lex, &dummy, true); ``` In that code one can see: if the subquery has table that has 'is_jtbm()==true', then ... some processing is done to re-compute its attributes after the table has been pulled up into the parent select. The same should be done for TABLE_LIST elements that are JSON_TABLE(..). If you'll introduce a new function, fix_after_pullout() is a good name. It is already used for Item-derived classes. > If not, i'll dig into it. > And you're welcome to observe the new patch. > > Best regards. > HF -- BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

2 1

[Maria-developers] MDEV-21829: Unique class: discussion followup part #2
by Sergey Petrunia 25 Sep '20

25 Sep '20

Hello Igor, Trying to capture the input from the last call. Currently class Unique is using RB-Tree for its in-memory storage. RB-Tree is implemented by struct TREE and tree_XXX() functions (further I'll refer to them both as just "TREE"). It is possible to get Unique to use some other data structure for its in-memory storage. The most obvious candidate is a hash table. (There are currently no plans to do this, though). So, your request was that - An interface is developed for in-memory storage (let's refer to it as Unique::MemStorage) - An implementation of this interface is developed that uses TREE. (We can't change the TREE itself as it is used in multple places). Is this really the intent? Please clarify. My reaction is that the above described change does not seem to have any relevance to MDEV-21829. MDEV-21829 is about supporting variable-length keys, not about changing, or preparing to change Unique::MemStorage. BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 1

Re: [Maria-developers] 2564334b4c1: Backported setting of transcation.on=1 in THD::reset_for_reuse()
by Sergei Golubchik 25 Sep '20

25 Sep '20

Hi, Michael! > diff --git a/sql/sql_class.cc b/sql/sql_class.cc > index 7327f270c33..8f6356b15c7 100644 > --- a/sql/sql_class.cc > +++ b/sql/sql_class.cc > @@ -1640,6 +1640,7 @@ void THD::reset_for_reuse() > abort_on_warning= 0; > free_connection_done= 0; > m_command= COM_CONNECT; > + transaction.on= 1; > #if defined(ENABLED_PROFILING) > profiling.reset(); > #endif This looks risky, to change transaction.on at some random point in time. I understand that it's not a random point and it should be actually safe here. But perhaps you can add an assert to document that it is safe? Like, transaction.on should be already 1 or there should be no active transaction. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 1

Re: [Maria-developers] fc07306120f: MDEV-23586 Mariabackup: GTID saved for replication in 10.4.14 is wrong
by Sergei Golubchik 25 Sep '20

25 Sep '20

Hi, Monty! Just a couple of minor comments Note, the below is from the combined diff of this commit and your earlier MDEV-21953 commit, so you can see some lines here that are not part of the fc07306120f. > diff --git a/sql/handler.cc b/sql/handler.cc > index cc7eedd18f2..402db310ed6 100644 > --- a/sql/handler.cc > +++ b/sql/handler.cc > @@ -133,6 +133,23 @@ static plugin_ref ha_default_tmp_plugin(THD *thd) > return ha_default_plugin(thd); > } > > +#if defined(WITH_ARIA_STORAGE_ENGINE) && MYSQL_VERSION_ID < 100500 > +void ha_maria_implicit_commit(THD *thd, bool new_trn) > +{ > + if (ha_maria::has_active_transaction(thd)) > + { > + int error; > + MDL_request mdl_request; > + mdl_request.init(MDL_key::BACKUP, "", "", MDL_BACKUP_COMMIT, MDL_EXPLICIT); > + error= thd->mdl_context.acquire_lock(&mdl_request, > + thd->variables.lock_wait_timeout); > + ha_maria::implicit_commit(thd, new_trn); > + if (!error) > + thd->mdl_context.release_lock(mdl_request.ticket); This looks rather fishy. You commit in aria unconditionally? whether you got the lock or not? > + } > +} > +#endif > + > > /** @brief > Return the default storage engine handlerton for thread > @@ -1753,13 +1777,14 @@ int ha_commit_one_phase(THD *thd, bool all) > if ((res= thd->wait_for_prior_commit())) > DBUG_RETURN(res); > } > - res= commit_one_phase_2(thd, all, trans, is_real_trans); > + res= commit_one_phase_2(thd, all, trans, is_real_trans, rw_trans); > DBUG_RETURN(res); > } > > > static int > -commit_one_phase_2(THD *thd, bool all, THD_TRANS *trans, bool is_real_trans) > +commit_one_phase_2(THD *thd, bool all, THD_TRANS *trans, bool is_real_trans, > + bool rw_trans) rw_trans seems to be unused here (also in ha_commit_one_phase()) > { > int error= 0; > uint count= 0; > diff --git a/sql/sql_class.cc b/sql/sql_class.cc > index 40e606425c5..71b447fb920 100644 > --- a/sql/sql_class.cc > +++ b/sql/sql_class.cc > @@ -1383,7 +1384,11 @@ void THD::update_all_stats() > void THD::init_for_queries() > { > set_time(); > - ha_enable_transaction(this,TRUE); > + /* > + We don't need to call ha_enable_transaction() as we can't have > + any active transactions that has to be commited > + */ > + transaction.on= TRUE; I just commented on the similar code in another commit of yours. don't just say "can't have any active transactions", add an assert instead, please. > > reset_root_defaults(mem_root, variables.query_alloc_block_size, > variables.query_prealloc_size); Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 1

Re: [Maria-developers] New patch on the JSON_TABLE.
by Sergey Petrunia 23 Sep '20

23 Sep '20

One more thing: On Mon, Sep 14, 2020 at 03:38:38PM +0400, Alexey Botchkov wrote: > CREATE TABLE t1(id INT, f1 JSON); > INSERT INTO t1 VALUES (1, '{\"1\": 1}'), (2, '{\"1\": 2}'), (3, '{\"1\": > 3}'), > (4, '{\"1\": 4}'), (5, '{\"1\": 5}'), (6, '{\"1\": 6}'); > > SELECT * FROM t1 WHERE id IN > (SELECT id FROM t1 as tt2, > JSON_TABLE(f1, "$" COLUMNS (jf FOR ORDINALITY)) AS tbl); I've noticed that Optimizer Trace is not saved for this query. I get MariaDB [test]> select * from information_schema.optimizer_trace\G *************************** 1. row *************************** QUERY: TRACE: MISSING_BYTES_BEYOND_MAX_MEM_SIZE: 0 INSUFFICIENT_PRIVILEGES: 1 1 row in set (0.001 sec) INSUFFICIENT_PRIVILEGES: 1! This is only supposed to happen when the query uses VIEWs which definition cannot be viewed by the user executing the query. Could you investigate and fix this please? BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

[Maria-developers] On bb-10.6-monty work
by Nikita Malyavin 16 Sep '20

16 Sep '20

Hello, Monty! I've been asked to review your recent work regarding memory footprint optimizations. I went through all the work done, and mostly liked the changes done. So here are only two commits i'd like to highlight > commit bf8bd0573440dc39490eb795b5455735dd63132b > Author: Monty <monty(a)mariadb.org> > Date: Tue Jul 14 18:36:05 2020 +0300 > > MDEV-23001 Precreate static Item_bool() to simplify code > > The following changes where done: > - Create global Item: Item_false and Item_true > - Replace all creation if 'FALSE' and 'TRUE' top level items used for > WHERE/HAVING/ON clauses to use Item_false and Item_true. > > The benefit are: > - Less and faster code > - No test needed if we where able to create the new item. > - Fixed possible errors if 'new' would have failed for the Item_bool's > > diff --git a/sql/item.h b/sql/item.h > index c4fbf8f9c0a..4cdee637415 100644 > --- a/sql/item.h > +++ b/sql/item.h > @@ -4236,6 +4239,23 @@ class Item_bool :public Item_int > }; > > > +class Item_bool_static :public Item_bool > +{ > +public: > + Item_bool_static(const char *str_arg, longlong i): > + Item_bool(NULL, str_arg, i) {}; > + > + /* Create dummy functions for things that may change the static item */ > + void fix_length_and_charset(uint32 max_char_length_arg, CHARSET_INFO *cs) > + {} > + void fix_char_length(size_t max_char_length_arg) > + {} > + void set_join_tab_idx(uint join_tab_idx_arg) > + { DBUG_ASSERT(0); } > +}; > + > +extern const Item_bool_static Item_false, Item_true; > + > class Item_uint :public Item_int > { > public: Do we really need an additional class for static Item_bool's? I know that we have constructors hidden for Item, so maybe just unhide it in Item_bool, and that's it. Besides, I see fix_char_length is not virtual at all, so reimplementing it has no efect. And why can't we do all that in Item_bool itself? Note that adding new class increases a virtual method table, and it's huge already, for Item hierarchy. We could also add some other frequently used Items, like 0, 1, '', NULL, to a static memory. > commit 6ca3ab6c2c18cdbf614984cb9d5312d58e718372 > Author: Monty <monty(a)mariadb.org> > Date: Wed Sep 2 03:13:32 2020 +0300 > > Split item->flags into base_flags and with_flags > > This was done to simplify copying of with_* flags > > Other things: > - Changed Flags to C++ enums, which enables gdb to print > out bit values for the flags. This also enables compiler > errors if one tries to manipulate a non existing bit in > a variable. > - Added set_maybe_null() as a shortcut as setting the > MAYBE_NULL flags was used in a LOT of places. > - Renamed PARAM flag to SP_VAR to ensure it's not confused with persistent > statement parameters. > > diff --git a/sql/item.h b/sql/item.h > index 09618bbb52c..9a794d96982 100644 > --- a/sql/item.h > +++ b/sql/item.h > @@ -717,28 +717,94 @@ class Item_const > > #define STOP_PTR ((void *) 1) > > -/* Bits used in Item.flags */ > -/* If item may be null */ > -#define ITEM_FLAG_MAYBE_NULL_SHIFT 0 > -#define ITEM_FLAG_MAYBE_NULL (1<<ITEM_FLAG_MAYBE_NULL_SHIFT) > -/* If used in GROUP BY list of a query with ROLLUP */ > -#define ITEM_FLAG_IN_ROLLUP (1<<1) > -/* If Item contains an SP parameter */ > -#define ITEM_FLAG_WITH_PARAM (1<<2) > -/* If item contains a window func */ > -#define ITEM_FLAG_WITH_WINDOW_FUNC (1<<3) > -/* True if any item except Item_sum contains a field. Set during parsing. */ > -#define ITEM_FLAG_WITH_FIELD (1<<4) > -/* If item was fixed with fix_fields */ > -#define ITEM_FLAG_FIXED (1<<5) > -/* Indicates that name of this Item autogenerated or set by user */ > -#define ITEM_FLAG_IS_AUTOGENERATED_NAME (1 << 6) > -/* Indicates that this item is in CYCLE clause of WITH */ > -#define ITEM_FLAG_IS_IN_WITH_CYCLE (1<<7) > -/* True if item contains a sum func */ > -#define ITEM_FLAG_WITH_SUM_FUNC (1<< 8) > -/* True if item containts a sub query */ > -#define ITEM_FLAG_WITH_SUBQUERY (1<< 9) > + > +/* Base flags (including IN) for an item */ > + > + > +typedef uint8 item_flags_t; > + > +enum class item_base_t : item_flags_t > +{ > + NONE= 0, > +#define ITEM_FLAGS_MAYBE_NULL_SHIFT 0 // Must match MAYBE_NULL > + MAYBE_NULL= (1<<0), // May be NULL. > + IN_ROLLUP= (1<<1), // Appears in GROUP BY list > + // of a query with ROLLUP. > + FIXED= (1<<2), // Was fixed with fix_fields(). > + IS_AUTOGENERATED_NAME= (1<<3), // The name if this Item was > + // generated or set by the user. > + IS_IN_WITH_CYCLE= (1<<4) // This item is in CYCLE clause > + // of WITH. > +}; > + > + > +/* Flags that tells us what kind of items the item contains */ > + > +enum class item_with_t : item_flags_t > +{ > + NONE= 0, > + SP_VAR= (1<<0), // If Item contains a stored procedure variable > + WINDOW_FUNC= (1<<1), // If item contains a window func > + FIELD= (1<<2), // If any item except Item_sum contains a field. > + SUM_FUNC= (1<<3), // If item contains a sum func > + SUBQUERY= (1<<4) // If item containts a sub query > +}; > + > + > +/* Make operations in item_base_t and item_with_t work like 'int' */ > +static inline item_base_t operator&(const item_base_t a, const item_base_t b) > +{ > + return (item_base_t) (((item_flags_t) a) & ((item_flags_t) b)); > +} > + > +static inline item_base_t & operator&=(item_base_t &a, item_base_t b) > +{ > + a= (item_base_t) (((item_flags_t) a) & (item_flags_t) b); > + return a; > +} > + > +static inline item_base_t operator|(const item_base_t a, const item_base_t b) > +{ > + return (item_base_t) (((item_flags_t) a) | ((item_flags_t) b)); > +} > + > +static inline item_base_t & operator|=(item_base_t &a, item_base_t b) > +{ > + a= (item_base_t) (((item_flags_t) a) | (item_flags_t) b); > + return a; > +} > + > +static inline item_base_t operator~(const item_base_t a) > +{ > + return (item_base_t) ~(item_flags_t) a; > +} > + > +static inline item_with_t operator&(const item_with_t a, const item_with_t b) > +{ > + return (item_with_t) (((item_flags_t) a) & ((item_flags_t) b)); > +} > + > +static inline item_with_t & operator&=(item_with_t &a, item_with_t b) > +{ > + a= (item_with_t) (((item_flags_t) a) & (item_flags_t) b); > + return a; > +} > + > +static inline item_with_t operator|(const item_with_t a, const item_with_t b) > +{ > + return (item_with_t) (((item_flags_t) a) | ((item_flags_t) b)); > +} > + > +static inline item_with_t & operator|=(item_with_t &a, item_with_t b) > +{ > + a= (item_with_t) (((item_flags_t) a) | (item_flags_t) b); > + return a; > +} > + > +static inline item_with_t operator~(const item_with_t a) > +{ > + return (item_with_t) ~(item_flags_t) a; > +} > > > class Item :public Value_source, > @@ -932,8 +998,8 @@ class Item :public Value_source, > const char *orig_name; > > /* All common bool variables for an Item is stored here */ > - typedef uint16 item_flags_t; > - item_flags_t flags; > + item_base_t base_flags; > + item_with_t with_flags; > > /* Marker is used in some functions to temporary mark an item */ > int16 marker; Nice experimentation! I wanted to suggest using constexprs inside Item, but this is even better, because you cant freely add a bit that not in the enum. Though I think it is too bloaty to specify all the operators each time, so I'd prefer to see duplications metaprogrammed out somehow. There is no sane way in C++ (even in modern one) to specify traits, which extend the class functionality (see Golang's traits), so I'd suggest to use good old preprocessor way, like following: #define T item_base_t #include "setup_bitwise_operators.inc" #define T item_with_t #include "setup_bitwise_operators.inc" and then in setup_bitwise_operators.inc: static inline T operator&(const T a, const T b) { return (T) (((item_flags_t) a) & ((item_flags_t) b)); } And so on. Another variant I've just googled is to make a template operator for all types, but restrict its applicatory type set with std::enable_if<T>: https://www.justsoftwaresolutions.co.uk/cplusplus/using-enum-classes-as-bit… So we would only need to add a structure: template<> struct enable_bitmask_operators<item_base_t>{ static constexpr bool enable=true; }; To work. Thereby, the first approach is: + Good old way + Works fast - not a C++ way, though we work in a C++-only file The second approach is: + modern + C++ way - may compile slow - may produce uncomfortable errors when compiler tries to deduce a function/operator to apply Up to you whic way to use, but I think it should be done, since we have many other flag sets, which are likely to be prettified with the same style. -- Yours truly, Nikita Malyavin

2 3

[Maria-developers] MDEV-21829:Followup: On Unique object and variable-size keys
by Sergey Petrunia 15 Sep '20

15 Sep '20

Hi Varun, I was looking at the code for MDEV-21829, trying to understand whether there are any issues with how Unique object was extended to handle variable-size keys. == TREE object (the RB tree) == TREE (the structure and functions that operate on it) support variable-sized keys, however the keys "are expected to know" their size. tree_insert() has "uint key_size" argument. This way, tree_insert() knows how many bytes to allocate and copy when putting the key into the tree. The rest of the TREE code doesn't seem to care about the size of the keys. When TREE code needs to compare a couple of keys, it will call TREE::compare callack. The callback only receives key pointers as arguments, that is, it is expected to infer key size from the contents of the key. The same happens when when walking the tree. Only "uchar *key" is passed to tree_walk_action callback. The walk function has to figure out they key size on its own. == Unique == Unique class cannot do the same what TREE functions function do. It needs to be aware of the sizes of the values it is storing: - It will need to pass value size to tree_insert() - It will need to know value size to write it into tmp file. - It will need to read it back when merging. How does it know it? Unique::merge() and Unique::walk() with merge_walk() functions do make certain assumptions about the data format being used. == The review points == Is this bad? I think tight coupling between the Unique object and the specifics of the data format it is storing is bad. What is worse is that currently this coupling is implicit, it is not immediately apparent to somebody just looking at the code. Possible ways out: 1. Explicitly declare in Unique class that variable-sized records must use certain data format. 2. Isolate the Unique from the specifics of the data format used. 2A. Let Unique get the size of the key as argument of unique_add() and then keep it together with the key. This creates some RAM/storage overhead. 2B. Pass a read_size() callback function which would get the size from the data? BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

2 1

Re: [Maria-developers] 099596296ce: Avoid mallocs when using LONGLONG_BUFFER_SIZE
by Sergei Golubchik 13 Sep '20

13 Sep '20

Hi, Michael! This is a good catch! But note that LONGLONG_BUFFER_SIZE already reserves +1 for '\0', it's defined as /* Max length needed for a buffer to hold a longlong or ulonglong + end \0 */ #define LONGLONG_BUFFER_SIZE 21 The problem is that String::set_int doesn't use LONGLONG_BUFFER_SIZE, it uses 20*cs->mbmaxlen+1 This +1 is supposed to reserve one byte for '\0'. Which is incorrect, because alloc() does it too, so a better fix would be to remove +1 from String::set_int(). On Sep 08, Michael Widenius wrote: > revision-id: 099596296ce (mariadb-10.5.2-272-g099596296ce) > parent(s): 2de0c74f94c > author: Michael Widenius <michael.widenius(a)gmail.com> > committer: Michael Widenius <michael.widenius(a)gmail.com> > timestamp: 2020-09-03 15:17:56 +0300 > message: > > Avoid mallocs when using LONGLONG_BUFFER_SIZE > > When using String::alloc() to allocate a string, the String ensures that > there is space for an extra NULL byte in the buffer and if not, reallocates > the string. This is a problem with the String::set_int() that calls > alloc(LONGLONG_BUFFER_SIZE), which forces an extra malloc/free to > happen. > > Fixed by using LONGLONG_BUFFER_SIZE+1 for StringBuffer's. > > diff --git a/sql/field.cc b/sql/field.cc > index ab7254e483d..2be4da8d4a8 100644 > --- a/sql/field.cc > +++ b/sql/field.cc > @@ -1085,7 +1085,7 @@ Field_longstr::make_packed_sort_key_part(uchar *buff, > uchar* > Field_longstr::pack_sort_string(uchar *to, const SORT_FIELD_ATTR *sort_field) > { > - StringBuffer<LONGLONG_BUFFER_SIZE> buf; > + StringBuffer<LONGLONG_BUFFER_SIZE+1> buf; > val_str(&buf, &buf); > return to + sort_field->pack_sort_string(to, &buf, field_charset()); > } > diff --git a/sql/item.cc b/sql/item.cc > index 4a26cb6c140..0d947749e01 100644 > --- a/sql/item.cc > +++ b/sql/item.cc > @@ -3612,7 +3612,7 @@ String *Item_int::val_str(String *str) > > void Item_int::print(String *str, enum_query_type query_type) > { > - StringBuffer<LONGLONG_BUFFER_SIZE> buf; > + StringBuffer<LONGLONG_BUFFER_SIZE+1> buf; > // my_charset_bin is good enough for numbers > buf.set_int(value, unsigned_flag, &my_charset_bin); > str->append(buf); > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 1