- developers - lists.mariadb.org

Re: [Maria-developers] [Commits] fb619954867: MDEV-15010: Wrong Seconds_Behind_Master when only starting the SQL_Thread
by andrei.elkin＠pp.inet.fi 16 Jul '19

16 Jul '19

Sujatha, howdy. Thanks for a good piece of work! The fact that it's actually an upstream contribution only pleases more :-), and to that regard we need a commit id to refer to (and to the people behind that work :-)). > revision-id: fb619954867232b3a733213a2703b2a425dff8e9 (mariadb-5.5.64-19-gfb619954867) > parent(s): 7a7d9904e12335ee8b1eea9671138b3c469a3829 > author: Sujatha > committer: Sujatha > timestamp: 2019-06-19 14:48:18 +0530 > message: > > MDEV-15010: Wrong Seconds_Behind_Master when only starting the SQL_Thread > > Problem: > ======= > When I run replication normally, I get a value in "Seconds_Behind_Master" from > "SHOW SLAVE STATUS". However, when I run only the SQL Thread (on local relay > logs that have been downloaded previously), I have "NULL" in > "Seconds_Behind_Master". I would expect to have a numeric value. > > Fix: > === > Implemented following changes. > case 1: "Seconds_Behind_Master" shows 0, when SQL thread is in sync with IO > thread and IO thread is running. > case 2: "Seconds_Behind_Master" reports NULL, when SQL thread is in sync with > IO thread and IO thread is stopped. > case 3: "Seconds_Behind_Master" reports NULL, when SQL thread is stopped while > IO thread is up and running. > case 4: "Seconds_Behind_Master" reports a valid numerical value when IO thread > is stopped and SQL thread is consuming existing relay log. I have a question to the case 4 in the tests actually. > > --- > .../suite/rpl/r/rpl_seconds_behind_master.result | 40 ++++++++++ > .../suite/rpl/t/rpl_seconds_behind_master.test | 92 ++++++++++++++++++++++ > sql/slave.cc | 82 ++++++++++++------- > 3 files changed, 187 insertions(+), 27 deletions(-) > > diff --git a/mysql-test/suite/rpl/r/rpl_seconds_behind_master.result b/mysql-test/suite/rpl/r/rpl_seconds_behind_master.result > new file mode 100644 > index 00000000000..1c0ce7c1b19 > --- /dev/null > +++ b/mysql-test/suite/rpl/r/rpl_seconds_behind_master.result > @@ -0,0 +1,40 @@ > +include/master-slave.inc > +[connection master] > +CREATE TABLE t1 (a INT PRIMARY KEY, b INT); > +****************************************************************** > +* Case1: SQL thread is in sync with IO thread and IO thread is UP > +* Seconds_Behind_Master should be '0'. > +****************************************************************** > +include/assert.inc [Seconds_Behind_Master should be 0] > +****************************************************************** > +* Case2: SQL thread is in sync with IO thread and IO thread is > +* stopped. > +* Seconds_Behind_Master should be 'NULL'. > +****************************************************************** > +include/stop_slave_io.inc > +include/assert.inc [Seconds_Behind_Master should be NULL] > +****************************************************************** > +* Case3: SQL thread is stopped while IO thread is up and running. > +* Seconds_Behind_Master should be 'NULL'. > +****************************************************************** > +START SLAVE IO_THREAD; > +include/wait_for_slave_io_to_start.inc > +include/stop_slave_sql.inc > +include/assert.inc [Seconds_Behind_Master should be NULL] > +****************************************************************** > +* Case4: IO thread is stopped. SQL thread is consuming existing relay > +* log. Hence the Seconds_Behind_Master should have a numeric > +* value > 0 > +****************************************************************** > +INSERT INTO t1 VALUES (1,sleep(2)); > +INSERT INTO t1 VALUES (2,sleep(2)); > +include/sync_slave_io_with_master.inc > +include/stop_slave_io.inc > +START SLAVE SQL_THREAD; > +include/wait_for_slave_sql_to_start.inc > +include/assert.inc [Seconds_Behind_Master should be > 0] > +START SLAVE IO_THREAD; > +include/wait_for_slave_io_to_start.inc > +******* CLEANUP ********* > +DROP TABLE t1; > +include/rpl_end.inc > diff --git a/mysql-test/suite/rpl/t/rpl_seconds_behind_master.test b/mysql-test/suite/rpl/t/rpl_seconds_behind_master.test > new file mode 100644 > index 00000000000..6f9ed2939e1 > --- /dev/null > +++ b/mysql-test/suite/rpl/t/rpl_seconds_behind_master.test > @@ -0,0 +1,92 @@ > +# ==== Purpose ==== > +# > +# Test verifies that Seconds_Behind_Master(SBM) values are as expected in various > +# scenario listed below. > +# > +# case 1: SBM=0 if SQL thread is in sync with IO thread and IO thread is UP > +# case 2: SBM=NULL if SQL thread is in sync with IO thread and IO thread is > +# stopped. > +# case 3: SBM=NULL if SQL thread is stopped while IO thread is up and running. > +# case 4: SBM > 0 and SBM != NULL if IO thread is stopped. SQL thread is > +# consuming existing relay log. > +# > +# ==== Implementation ==== > +# > +# Steps: > +# 1 - Create a table and sync it with slave and verify SBM value. > +# 2 - Now stop IO thread and verify SBM > +# 3 - Start IO thread, Stop SQL thread and verify SBM > +# 4 - Execute slow INSERT statements with SLEEP(2) on master. Ensure that > +# the IO thread has read these INSERT statements and then bring it down. > +# 5 - Start the SQL thread and verify that while it is trying to consume > +# these slow INSERT statements in relay log the SBM value is > 0. > +# > +# ==== References ==== > +# > +# MDEV-15010: Wrong Seconds_Behind_Master when only starting the SQL_Thread. > + > +--source include/have_binlog_format_statement.inc > +--source include/master-slave.inc > + > +--disable_query_log > +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); > +--enable_query_log > + > +CREATE TABLE t1 (a INT PRIMARY KEY, b INT); > +--sync_slave_with_master > + > +-- echo ****************************************************************** > +-- echo * Case1: SQL thread is in sync with IO thread and IO thread is UP > +-- echo * Seconds_Behind_Master should be '0'. > +-- echo ****************************************************************** > +--let $assert_cond= [SHOW SLAVE STATUS, Seconds_Behind_Master, 1] = 0 > +--let $assert_text= Seconds_Behind_Master should be 0 > +--source include/assert.inc > + > +-- echo ****************************************************************** > +-- echo * Case2: SQL thread is in sync with IO thread and IO thread is > +-- echo * stopped. > +-- echo * Seconds_Behind_Master should be 'NULL'. > +-- echo ****************************************************************** > +--source include/stop_slave_io.inc > +--let $assert_cond= "[SHOW SLAVE STATUS, Seconds_Behind_Master, 1]" = "NULL" > +--let $assert_text= Seconds_Behind_Master should be NULL > +--source include/assert.inc > + > +-- echo ****************************************************************** > +-- echo * Case3: SQL thread is stopped while IO thread is up and running. > +-- echo * Seconds_Behind_Master should be 'NULL'. > +-- echo ****************************************************************** > +START SLAVE IO_THREAD; > +--source include/wait_for_slave_io_to_start.inc > +--source include/stop_slave_sql.inc > +--let $assert_cond= "[SHOW SLAVE STATUS, Seconds_Behind_Master, 1]" = "NULL" > +--let $assert_text= Seconds_Behind_Master should be NULL > +--source include/assert.inc > + > +-- echo ****************************************************************** > +-- echo * Case4: IO thread is stopped. SQL thread is consuming existing relay > +-- echo * log. Hence the Seconds_Behind_Master should have a numeric > +-- echo * value > 0 > +-- echo ****************************************************************** > +--connection master > +--disable_warnings > +INSERT INTO t1 VALUES (1,sleep(2)); > +INSERT INTO t1 VALUES (2,sleep(2)); > +--enable_warnings > +--source include/sync_slave_io_with_master.inc > +--source include/stop_slave_io.inc > + > +START SLAVE SQL_THREAD; > +--source include/wait_for_slave_sql_to_start.inc There's a theoretical (and even though I think it would hardly ever be practical) chance for the SQL thread to catch up while for some concurrency reason the waiting would take longer that it's naturally expected. How about to + LOCK TABLES t1 WRITE before START SLAVE SQL_THREAD and unlock > +--let $assert_cond= [SHOW SLAVE STATUS, Seconds_Behind_Master, 1] > 0 > +--let $assert_text= Seconds_Behind_Master should be > 0 > +--source include/assert.inc after the assert check? This measure would speak rather explicitly that the check is done just before any first event has been processed by the SQL thread, and as you prepare 2, then no synchronization is guaranteed. Cheers, Andrei > +START SLAVE IO_THREAD; > +--source include/wait_for_slave_io_to_start.inc > + > +--echo ******* CLEANUP ********* > +--connection master > +DROP TABLE t1; > + > +--source include/rpl_end.inc > diff --git a/sql/slave.cc b/sql/slave.cc > index a8946c69d18..7b4b7bd857e 100644 > --- a/sql/slave.cc > +++ b/sql/slave.cc > @@ -2192,38 +2192,66 @@ bool show_master_info(THD* thd, Master_info* mi) > protocol->store(mi->ssl_cert, &my_charset_bin); > protocol->store(mi->ssl_cipher, &my_charset_bin); > protocol->store(mi->ssl_key, &my_charset_bin); > - > /* > - Seconds_Behind_Master: if SQL thread is running and I/O thread is > - connected, we can compute it otherwise show NULL (i.e. unknown). > + The pseudo code to compute Seconds_Behind_Master: > + if (SQL thread is running) > + { > + if (SQL thread processed all the available relay log) > + { > + if (IO thread is running) > + print 0; > + else > + print NULL; > + } > + else > + compute Seconds_Behind_Master; > + } > + else > + print NULL; > */ > - if ((mi->slave_running == MYSQL_SLAVE_RUN_CONNECT) && > - mi->rli.slave_running) > + if (mi->rli.slave_running) > { > - long time_diff= ((long)(time(0) - mi->rli.last_master_timestamp) > - - mi->clock_diff_with_master); > /* > - Apparently on some systems time_diff can be <0. Here are possible > - reasons related to MySQL: > - - the master is itself a slave of another master whose time is ahead. > - - somebody used an explicit SET TIMESTAMP on the master. > - Possible reason related to granularity-to-second of time functions > - (nothing to do with MySQL), which can explain a value of -1: > - assume the master's and slave's time are perfectly synchronized, and > - that at slave's connection time, when the master's timestamp is read, > - it is at the very end of second 1, and (a very short time later) when > - the slave's timestamp is read it is at the very beginning of second > - 2. Then the recorded value for master is 1 and the recorded value for > - slave is 2. At SHOW SLAVE STATUS time, assume that the difference > - between timestamp of slave and rli->last_master_timestamp is 0 > - (i.e. they are in the same second), then we get 0-(2-1)=-1 as a result. > - This confuses users, so we don't go below 0: hence the max(). > - > - last_master_timestamp == 0 (an "impossible" timestamp 1970) is a > - special marker to say "consider we have caught up". > + Check if SQL thread is at the end of relay log > + Checking should be done using two conditions > + condition1: compare the log positions and > + condition2: compare the file names (to handle rotation case) > */ > - protocol->store((longlong)(mi->rli.last_master_timestamp ? > - max(0, time_diff) : 0)); > + if ((mi->master_log_pos == mi->rli.group_master_log_pos) && > + (!strcmp(mi->master_log_name, mi->rli.group_master_log_name))) > + { > + if (mi->slave_running == MYSQL_SLAVE_RUN_CONNECT) > + protocol->store(0LL); > + else > + protocol->store_null(); > + } > + else > + { > + long time_diff = ((long)(time(0) - mi->rli.last_master_timestamp) > + - mi->clock_diff_with_master); > + /* > + Apparently on some systems time_diff can be <0. Here are possible > + reasons related to MySQL: > + - the master is itself a slave of another master whose time is ahead. > + - somebody used an explicit SET TIMESTAMP on the master. > + Possible reason related to granularity-to-second of time functions > + (nothing to do with MySQL), which can explain a value of -1: > + assume the master's and slave's time are perfectly synchronized, and > + that at slave's connection time, when the master's timestamp is read, > + it is at the very end of second 1, and (a very short time later) when > + the slave's timestamp is read it is at the very beginning of second > + 2. Then the recorded value for master is 1 and the recorded value for > + slave is 2. At SHOW SLAVE STATUS time, assume that the difference > + between timestamp of slave and rli->last_master_timestamp is 0 > + (i.e. they are in the same second), then we get 0-(2-1)=-1 as a result. > + This confuses users, so we don't go below 0: hence the max(). > + > + last_master_timestamp == 0 (an "impossible" timestamp 1970) is a > + special marker to say "consider we have caught up". > + */ > + protocol->store((longlong)(mi->rli.last_master_timestamp ? > + max(0, time_diff) : 0)); > + } > } > else > { > _______________________________________________ > commits mailing list > commits(a)mariadb.org > https://lists.askmonty.org/cgi-bin/mailman/listinfo/commits

1 0

[Maria-developers] MDEV-8306: more input
by Sergey Petrunia 16 Jul '19

16 Jul '19

Hi Varun, Please find more input, I'm commenting on the second patch (the review is probably incomplete, I've made just one pass so might have missed something) > diff -urpN '--exclude=.*' 10.5-mdev8306-orig/sql/opt_subselect.cc 10.5-mdev8306/sql/opt_subselect.cc > --- 10.5-mdev8306-orig/sql/opt_subselect.cc 2019-07-10 11:30:41.151696943 +0300 > +++ 10.5-mdev8306/sql/opt_subselect.cc 2019-07-16 13:07:13.948467315 +0300 > @@ -5517,6 +5517,31 @@ enum_nested_loop_state join_tab_executio > sjm->materialized= TRUE; > } > } > + /* > + This should be the place to add the sub_select call for the > + prefix of the join order > + */ > + > + else if (tab->is_order_nest) > + { > + enum_nested_loop_state rc; > + JOIN *join= tab->join; > + NEST_INFO *nest_info= join->order_nest_info; > + > + if (!nest_info->materialized) > + { > + JOIN_TAB *join_tab= join->join_tab + join->const_tables; > + JOIN_TAB *save_return_tab= join->return_tab; > + if ((rc= sub_select(join, join_tab, FALSE)) < 0 || > + (rc= sub_select(join, join_tab, TRUE)) < 0) > + { > + join->return_tab= save_return_tab; > + DBUG_RETURN(rc); > + } > + join->return_tab= save_return_tab; > + nest_info->materialized= TRUE; > + } Looks a bit repetitive with the SJM nest code above, but this is not a big deal. A question: if the JOIN_TABs for the tables inside the sort nest are still in the join->join_tab array, where is the code to "jump over" them in regular join execution? > + } > > DBUG_RETURN(NESTED_LOOP_OK); > } ... > diff -urpN '--exclude=.*' 10.5-mdev8306-orig/sql/sql_class.h 10.5-mdev8306/sql/sql_class.h > --- 10.5-mdev8306-orig/sql/sql_class.h 2019-07-16 13:10:05.448491384 +0300 > +++ 10.5-mdev8306/sql/sql_class.h 2019-07-16 13:07:13.948467315 +0300 > @@ -6037,6 +6037,24 @@ public: > Copy_field *copy_field; /* Needed for SJ_Materialization scan */ > }; > > +class NEST_INFO : public Sql_alloc > +{ > +public: > + ~NEST_INFO() > + { > + nest_tab= NULL; > + table= NULL; > + } What is this for? Object members should not be used after destructor is called. If you're concerned about accidental access, please use TRASH_XXX macros to fill the variables with "already-freed" marker. > + TMP_TABLE_PARAM tmp_table_param; > + List<Item> nest_base_table_cols; > + List<Item> nest_temp_table_cols; > + TABLE *table; > + st_join_table *nest_tab; > + uint n_tables; > + bool materialized; /* TRUE <=> materialization already performed */ > + table_map nest_tables_map; > +}; > + > > /* Structs used when sorting */ > struct SORT_FIELD_ATTR ... > diff -urpN '--exclude=.*' 10.5-mdev8306-orig/sql/sql_select.cc 10.5-mdev8306/sql/sql_select.cc > --- 10.5-mdev8306-orig/sql/sql_select.cc 2019-07-16 13:10:05.452491478 +0300 > +++ 10.5-mdev8306/sql/sql_select.cc 2019-07-16 13:07:13.952467409 +0300 > return retval; > } > > +void substitute_base_to_nest_items(JOIN *join) > +{ > + NEST_INFO *order_nest_info= join->order_nest_info; > + if (!order_nest_info) > + return; > + REPLACE_NEST_FIELD_ARG arg= {join}; > + join->conds= join->conds->transform(join->thd, &Item::replace_with_nest_items, > + (uchar *) &arg); > + > + List_iterator<Item> it(join->fields_list); > + Item *item, *new_item; > + uint i=0; > + while ((item= it++)) > + { > + if ((new_item= item->transform(join->thd, > + &Item::replace_with_nest_items, > + (uchar *) &arg)) != item) > + it.replace(new_item); > + } Ok, so you are processing the select list, and the WHERE condition. What about the ON expressions, and the right sides of ref access? Also, how do we solve the issue of the same Item_field object being used in two conditions at once: - one inside the sort nest - one outside the sort nest (remember the example with " (inner_tbl_cond1 AND outer_tbl_con1) OR (inner_tbl_cond2 AND outer_tbl_cond2) " > +} > + > > /** ... > @@ -12136,6 +12314,35 @@ end_sj_materialize(JOIN *join, JOIN_TAB > } > > > +enum_nested_loop_state > +end_nest_materialization(JOIN *join, JOIN_TAB *join_tab, bool end_of_records) > +{ > + int error; > + THD *thd= join->thd; > + NEST_INFO *nest_info= join->order_nest_info; > + DBUG_ENTER("end_sj_materialize"); Please change to match the function name. > + if (!end_of_records) > + { > + TABLE *table= nest_info->table; > + fill_record(thd, table, table->field, > + nest_info->nest_base_table_cols, TRUE, FALSE); > + > + if (unlikely(thd->is_error())) > + DBUG_RETURN(NESTED_LOOP_ERROR); /* purecov: inspected */ > + if (unlikely((error= table->file->ha_write_tmp_row(table->record[0])))) > + { > + /* create_myisam_from_heap will generate error if needed */ > + if (table->file->is_fatal_error(error, HA_CHECK_DUP) && Is this check necessary? The table doesn't have any unique keys, why do we ignore duplicates? > + create_internal_tmp_table_from_heap(thd, table, > + nest_info->tmp_table_param.start_recinfo, > + &nest_info->tmp_table_param.recinfo, > + error, 1, NULL)) > + DBUG_RETURN(NESTED_LOOP_ERROR); /* purecov: inspected */ > + } > + } > + DBUG_RETURN(NESTED_LOOP_OK); > +} > + > /* > Check whether a join buffer can be used to join the specified table > BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

Re: [Maria-developers] d7b274fa257: MDEV-16932: ASAN heap-use-after-free in my_charlen_utf8 / my_well_formed_char_length_utf8 on 2nd execution of SP with ALTER trying to add bad CHECK
by Sergei Golubchik 16 Jul '19

16 Jul '19

Hi, Oleksandr! A couple of minor comments, see below On Jul 15, Oleksandr Byelkin wrote: > revision-id: d7b274fa257 (mariadb-10.2.25-63-gd7b274fa257) > parent(s): 64900e3d7c3 > author: Oleksandr Byelkin <sanja(a)mariadb.com> > committer: Oleksandr Byelkin <sanja(a)mariadb.com> > timestamp: 2019-07-11 13:29:51 +0200 > message: > > MDEV-16932: ASAN heap-use-after-free in my_charlen_utf8 / my_well_formed_char_length_utf8 on 2nd execution of SP with ALTER trying to add bad CHECK > > Make automatic name generation during execution (not prepare). > > Check result of memory allocation operation. > > diff --git a/sql/sql_table.cc b/sql/sql_table.cc > index 043cfaaaaaa..636fb9f427c 100644 > --- a/sql/sql_table.cc > +++ b/sql/sql_table.cc > @@ -6205,6 +6210,10 @@ handle_if_exists_options(THD *thd, TABLE *table, Alter_info *alter_info) > Virtual_column_info *check; > TABLE_SHARE *share= table->s; > uint c; > + > + if (fix_constraints_names(thd, &alter_info->check_constraint_list)) > + DBUG_RETURN(TRUE); It's not very logical, I think, to generate constraint names in handle_if_exists_options(), I'd rather move it out and put directly in mysql_alter_table(). > + > while ((check=it++)) > { > if (!(check->flags & Alter_info::CHECK_CONSTRAINT_IF_NOT_EXISTS) && > @@ -6232,10 +6241,44 @@ handle_if_exists_options(THD *thd, TABLE *table, Alter_info *alter_info) > } > } > > - DBUG_VOID_RETURN; > + DBUG_RETURN(FALSE); > } > > > +static bool fix_constraints_names(THD *thd, List<Virtual_column_info> > + *check_constraint_list) > +{ > + List_iterator<Virtual_column_info> it((*check_constraint_list)); > + Virtual_column_info *check; > + uint nr= 1; > + DBUG_ENTER("fix_constraints_names"); > + if (!check_constraint_list) > + DBUG_RETURN(FALSE); > + // Prevent accessing freed memory during generating unique names > + while ((check=it++)) > + { > + if (check->automatic_name) > + { > + check->name.str= NULL; > + check->name.length= 0; > + } > + } > + it.rewind(); > + // Grnerate unique names if needed typo > + while ((check=it++)) > + { > + if (!check->name.length) > + { > + check->automatic_name= TRUE; > + if (make_unique_constraint_name(thd, &check->name, > + check_constraint_list, > + &nr)) > + DBUG_RETURN(TRUE); > + } > + } > + DBUG_RETURN(FALSE); I would remove the first while() at all and just do the second one till the end: bool ret= FALSE; while ((check=it++)) if (check->automatic_name) ret |= make_unique_constraint_name(...); return ret; Because thd->strmake() basically never fails, it doesn't make much sense to optimize for a case when it does. This code above assumes you set automatic_name=TRUE in mysql_prepare_create_table(). > +} > + > /** > Get Create_field object for newly created table by field index. Regards, Sergei

2 1

Re: [Maria-developers] Interaction between rpl_slave_state and rpl_binlog_state
by Kristian Nielsen 15 Jul '19

15 Jul '19

Hi Andrei! The @@gtid_current_pos exists for one sole purpose. This is to let the user promote a slave as the new master and attach the old master as a slave to the new master. By using master_use_gtid=current_pos, the exact same command can be used to attach a slave to the new master, regardless of whether that slave was previously a slave or a master: CHANGE MASTER TO master_host=new_promoted_master If not using gtid_current_pos (ie master_use_gtid=slave_pos), then to let the old master become a slave of the new master, the old master's position must explicitly be set: SET GLOBAL gtid_slave_pos=@@gtid_binlog_pos This is because for efficiency reasons, the master doesn't update the mysql.gtid_slave_pos in each commit. So now we can see why only GTIDs with the servers own server_id should contribute to @@gtid_current_pos. If a GTID was replicated from another server, that GTID will appear in the @@gtid_slave_pos. If the GTID originated on this server, it will appear in @@gtid_binlog_pos. The @@gtid_current_pos is the @@gtid_slave_pos extended with GTIDs originating on this server, hence only GTIDs with our own server id. Normally, every GTID in the binlog with a different server id than our own will already be in the @@gtid_slave_pos as well - since it originated on another server and was replicated to this server. Thus, in the normal case, where user did not play tricks with the binlog and slave state, extending the @@gtid_current_pos as suggested in MDEV-18404 has no effect - the GTIDs are already in the @@gtid_slave_pos, so @@gtid_current_pos is unaffected. And in case the user deliberately modified the state, it should be up to the user to decide what goes into @@gtid_slave_state and @@gtid_binlog_state. For example, the MDEV-18404 change would make it impossible on a server to remove a replicated GTID from the @@gtid_current_pos if --log-slave-updates (without the drastic RESET MASTER). Another problem is that the server cannot reliably compare GTIDs with distinct server ids to decide which one is the most recent. There is no guarantee that sequence numbers are monotonic across different server ids. Thus the MDEV-18404 method could create completely invalid positions in some setups where @@gtid_strict_mode=0 and replication domains are not strictly maintained. I don't see the value in MDEV-18404. If the user is updating @@gtid_binlog_state (itself a very drastic operation), and wants a specific GTID to go into the slave position - just update the @@gtid_slave_pos with the desired GTID, don't leave the server with an inconsistent replication state. And finally, let me reiterate: I consider the @@gtid_current_pos a design mistake. Better to just transfer the @@gtid_binlog_pos to the @@gtid_slave_pos _only_ at the point where an old master is turned into a slave. This can be done manually already, and it would be simple to implement automatic support for this with an extra option for CHANGE MASTER. Hope this helps, - Kristian. Andrei Elkin <andrei.elkin(a)mariadb.com> writes: > Kristian, howdy! > > I thought it's good to refer to a previous discussion that touched > 'current_pos' feature now when we need to resolve MDEV-18404 which I > look at under a specific angle, narrated here. > >> Finally, experience has shown that a _lot_ of users get problems when >> locally done transactions on a slave influence the slave's GTID position. In >> retrospect, I have realised that CHANGE MASTER TO >> master_use_gtid=current_pos was a mistake, only slave_pos should be used. >> Similarly, if a local transaction in a slave's binlog can cause transactions >> from the master to be silently ignored, it will cause a lot of grief for >> users. > > > To the definition of > > gtid_current_pos > > the docs https://mariadb.com/kb/en/library/gtid/#gtid_current_pos first introduce an intuition > with > > This variable is the GTID of the last change to the database for > each replication domain. Such changes can either be master events > (ie. local changes made by user or application), > or replicated events originating from another master server. > > and then define the variable essentially as an union of two sets: > > For each replication domain, if the server ID of the corresponding > GTID in @@gtid_binlog_pos is equal to the servers own server_id, and > the sequence number is higher than the corresponding GTID in > @@gtid_slave_pos, then the GTID from @@gtid_binlog_pos will be > used. Otherwise the GTID from @@gtid_slave_pos will be used for that > domain. > > The union operation is on a *subset* of @@gtid_binlog_pos (call it B') > and the whole @@gtid_slave_pos (S): > > C = B' \cup S > > where C @@gtid_current_pos. > The @@gtid_binlog_pos subset > consists of GTID:s originated by @@global.server_id (that is foreign > ones excluded). > > Notice that makes the union non symmetric as binary operator. A foreign > GTID feels perfectly fine in S and gets into the union result. > > The following use case > > --connection master > CREATE TABLE t (a INT); > RESET MASTER; > SELECT @@global.server_id; > # => 1 > SET @@server_id= 3; > INSERT INTO t SET a=3; > select @@global.gtid_binlog_state = '0-3-1'; > # => 1 > > technically it's a 'local changes', but when S (@@global.gtid_slave_pos) is empty > > SELECT @@global_current_pos = ''; > # => 1 > > This result is another way to read MDEV-18404. > > After thinking over the restricted definition of @@gtid_current_pos I could not arrive at > relaxing it to account the whole @@gtid_binlog_set which fixes everything. > I wonder what do think about this case, have I missed any use case that might have forced > to the definition be as it is? > > Below is a straightforward patch that sustains the current mtr rpl > suites, expect rpl_gtid_startpos (that exactly relies at one point on > the empty @@gtid_current_pos due to the union specifics, I am leaving > out changes to the test at this point). > > Cheers, > > Andrei > > > diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc > index 752b4172b6e..1f9802bd8f8 100644 > --- a/sql/rpl_gtid.cc > +++ b/sql/rpl_gtid.cc > @@ -845,8 +845,7 @@ rpl_slave_state::iterate(int (*cb)(rpl_gtid *, void *), void *data, > my_hash_init(&gtid_hash, &my_charset_bin, 32, offsetof(rpl_gtid, domain_id), > sizeof(uint32), NULL, NULL, HASH_UNIQUE); > for (i= 0; i < num_extra; ++i) > - if (extra_gtids[i].server_id == global_system_variables.server_id && > - my_hash_insert(&gtid_hash, (uchar *)(&extra_gtids[i]))) > + if (my_hash_insert(&gtid_hash, (uchar *)(&extra_gtids[i]))) > goto err; > > mysql_mutex_lock(&LOCK_slave_state);

1 0

Re: [Maria-developers] [Commits] 7cabdc461b2: MDEV-6860 Parallel async replication hangs on a Galera node
by Kristian Nielsen 15 Jul '19

15 Jul '19

sachin.setiya(a)mariadb.com writes: > revision-id: 7cabdc461b24fdebe599799d7964efa4b53815e3 (mariadb-10.1.39-91-g7cabdc461b2) > > MDEV-6860 Parallel async replication hangs on a Galera node > > Wait for previous commit beore preparing next transation for galera > diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc > index 8fef2d66635..7d38c36b840 100644 > --- a/sql/rpl_parallel.cc > +++ b/sql/rpl_parallel.cc > @@ -1181,7 +1181,7 @@ handle_rpl_parallel_thread(void *arg) > before, then wait now for the prior transaction to complete its > commit. > */ > - if (rgi->speculation == rpl_group_info::SPECULATE_WAIT && > + if ((rgi->speculation == rpl_group_info::SPECULATE_WAIT || WSREP_ON) && > (err= thd->wait_for_prior_commit())) Ouch! That's killing _all_ parallel replication when WSREP_ON :-/ Do you really need to do this? It seems quite a restriction for replicating to Galera if parallel replication is not allowed. (I wonder if this isn't just another symptom of the underlying problem that Galera has never been integrated properly into MariaDB and the group commit algorithm / transaction master?). But if the goal is to disable parallel replication in Galera, then you shouldn't do this, it will just confuse/disappoint users, and it will be slower than just using single-threaded replication. Instead, give an error if parallel replication and Galera is enabled at the same time, so users will know of the restriction. - Kristian.

2 2

[Maria-developers] Weekly Report
by Rucha Deodhar 15 Jul '19

15 Jul '19

Report for week 7: Hello! This week I worked on the initial review for INSERT...RETURNING. I removed some lines from my code, used select_result* instead of using with_returning_list, improved formatting, restored the structure of SQL_I_List by removing *saved_first, moved returning_list to LEX so that current_select and lex->first_select_lex() is not used, fixed the logic of my_ok() and send_eof(), and made some part more readable. Now INSERT...IGNORE is working too. For tests, I merged some test cases, removed the other ones and added some cases for existing tests. I also read about bison and how to use its stack to return items so that it can be used instead of list swapping. As suggested, I am making the list rule return the List<Item> as a return value, then the rule that uses select_item_list uses something like this: lex->returning=$n I have started implementing it but getting errors during build. Initially I was getting error: opt_select_expressions has no declared type when I tried making the entire rule return item_list. To fix that I added opt_select_expressions to %type <item_list> I am still trying to fix other errors. Could you give me feedback whether I am moving along the right path, have I missed something? opt_select_expressions: /*empty*/ {$$=NULL} | RETURNING select_item_list { //Make this entire rule return item_list something like: $$=lex->current_select->item_list } ; Insert parser: insert_field_spec opt_insert_update opt_select_expressions { something like if ($3) Lex->returning_list= $3; Lex->pop_select(); //main select if (Lex->check_main_unit_semantics()) MYSQL_YYABORT; } ; Regards, Rucha Deodhar <https://www.avast.com/en-in/recommend?utm_medium=email&utm_source=link&utm_…> I’m protected online with Avast Free Antivirus. Get it here — it’s free forever. <https://www.avast.com/en-in/recommend?utm_medium=email&utm_source=link&utm_…> <#DAB4FAD8-2DD7-40BB-A1B8-4E2AA1F9FDF2>

1 0

[Maria-developers] MDEV-8306: Some input on the code so far
by Sergey Petrunia 10 Jul '19

10 Jul '19

Hi Varun, Please find below some input on the current patch: > diff -urpN '--exclude=.*' 10.5-mdev8306-orig/sql/sql_class.h 10.5-mdev8306/sql/sql_class.h > --- 10.5-mdev8306-orig/sql/sql_class.h 2019-07-10 11:34:00.256256952 +0300 > +++ 10.5-mdev8306/sql/sql_class.h 2019-07-10 11:34:15.436604510 +0300 > @@ -6037,6 +6037,15 @@ public: > Copy_field *copy_field; /* Needed for SJ_Materialization scan */ > }; > > +class NEST_INFO : public Sql_alloc Needs better naming, there are all kinds of nests. Making the first attempt: SORT_NEST_INFO ? > +{ > +public: > + TMP_TABLE_PARAM tmp_table_param; > + List<Item> nest_table_cols; > + TABLE *table; > + uint n_tables; > +}; > + > > /* Structs used when sorting */ > struct SORT_FIELD_ATTR > diff -urpN '--exclude=.*' 10.5-mdev8306-orig/sql/sql_select.cc 10.5-mdev8306/sql/sql_select.cc > --- 10.5-mdev8306-orig/sql/sql_select.cc 2019-07-10 11:34:00.260257044 +0300 > +++ 10.5-mdev8306/sql/sql_select.cc 2019-07-10 11:34:15.448604784 +0300 > @@ -8135,6 +8143,32 @@ choose_plan(JOIN *join, table_map join_t > use_cond_selectivity)) > DBUG_RETURN(TRUE); > } > + trace_plan.end(); > + > + for (uint tablenr=0;tablenr < join->table_count;tablenr++) > + { > + POSITION *pos= &join->best_positions[tablenr]; > + join->order_nest_tables|= pos->table->table->map; > + if (pos->ordering_achieved) > + { > + join->order_nest= TRUE; > + break; > + } > + } The above looks like query plan refinement action, very close to what is done in get_best_combination. Can this code be moved into that function? > + > + if (join->order_nest && unlikely(thd->trace_started())) > + { > + Json_writer_array trace_order_nest(thd, "order_nest"); > + > + for (uint tablenr=0;tablenr < join->table_count;tablenr++) > + { > + POSITION *pos= &join->best_positions[tablenr]; > + trace_order_nest.add_table_name(pos->table); > + if (pos->ordering_achieved) > + break; > + } > + } > + > > /* > Store the cost of this query into a user variable ... > @@ -14041,6 +14186,132 @@ ORDER *simple_remove_const(ORDER *order, > } > > > +/* > + This function basically tries to propgate all the multiple equalites > + for the Field items, so that one can use them to generate QEP that would > + also take into consideration equality propagation > +*/ > +void propagate_equal_field_for_orderby(JOIN *join, ORDER *first_order) > +{ > + ORDER *order; > + table_map not_const_tables= ~join->const_table_map; > + for (order= first_order; order; order= order->next) > + { > + table_map order_tables=order->item[0]->used_tables(); > + if (!(order_tables & not_const_tables)) > + continue; Is the above necessary? I think we have an optimization which removes constant element from ORDER BY list. (Perhaps, that optimization is invoked after the join optimizer. If that is the case, it should be done before the join optimizer) > + else > + { > + if (optimizer_flag(join->thd, OPTIMIZER_SWITCH_ORDERBY_EQ_PROP) && > + order->item[0]->real_item()->type() == Item::FIELD_ITEM && > + join->cond_equal) > + { > + Item *item= order->item[0]; > + /* > + TODO: equality substitution in the context of ORDER BY is > + sometimes allowed when it is not allowed in the general case. > + We make the below call for its side effect: it will locate the > + multiple equality the item belongs to and set item->item_equal > + accordingly. > + */ > + (void)item->propagate_equal_fields(join->thd, > + Value_source:: > + Context_identity(), > + join->cond_equal); > + } > + } > + } > +} > + > +/* > + This function basicall checks if by considering the current join_tab > + would we be able to achieve the ordering > +*/ > + > +bool check_join_prefix_contains_ordering(JOIN *join, JOIN_TAB *tab, > + table_map previous_tables) > +{ > + ORDER *order; > + table_map not_const_tables= ~join->const_table_map; > + for (order= join->order; order; order= order->next) > + { > + table_map order_tables=order->item[0]->used_tables(); > + if (!(order_tables & not_const_tables)) > + continue; Same question as above - at this stage ORDER BY list should not have const elements. > + else > + { > + Item_equal *item_eq= order->item[0]->get_item_equal(); > + if (((previous_tables | tab->table->map) & order_tables) == order_tables || > + (item_eq && (item_eq->used_tables() & (previous_tables | tab->table->map)))) > + continue; > + else > + return FALSE; > + } What about handling for non-const items that are not members of Item_equal? (e.g. ORDER BY left(table.col, 3)) ?) > + } > + return TRUE; > +} > + > + > +bool setup_order_nest(JOIN *join, JOIN_TAB *tab) (Same question about how to call the nest. I would call it "sort nest") > +{ > + if (!tab->is_order_nest) > + return FALSE; > + THD *thd= join->thd; > + Field_iterator_table field_iterator; > + > + JOIN_TAB *start_tab= join->join_tab+join->const_tables, *j; > + NEST_INFO* order_nest_info= join->order_nest_info; > + > + /* This needs to be added to JOIN structure, looks the best option or we > + can have a seperate struture NEST_INFO to hold it > + */ > + > + for (j= start_tab; j < tab; j++) > + { > + TABLE *table= start_tab->table; > + field_iterator.set_table(table); > + for (; !field_iterator.end_of_fields(); field_iterator.next()) > + { > + Field *field= field_iterator.field(); > + if (!bitmap_is_set(table->read_set, field->field_index)) > + continue; > + Item *item; > + if (!(item= field_iterator.create_item(thd))) > + return TRUE; > + order_nest_info->nest_table_cols.push_back(item, thd->mem_root); > + } > + } > + > + /* > + TODO also add ORDER ITEMS that are expressions, only fields are covered above > + */ > + DBUG_ASSERT(!tab->table); > + > + order_nest_info->tmp_table_param.init(); > + order_nest_info->tmp_table_param.bit_fields_as_long= TRUE; > + order_nest_info->tmp_table_param.field_count= order_nest_info->nest_table_cols.elements; > + order_nest_info->tmp_table_param.force_not_null_cols= FALSE; > + > + const LEX_CSTRING order_nest_name= { STRING_WITH_LEN("order-nest") }; > + if (!(tab->table= create_tmp_table(thd, &order_nest_info->tmp_table_param, > + order_nest_info->nest_table_cols, (ORDER*) 0, > + FALSE /* distinct */, > + 0, /*save_sum_fields*/ > + thd->variables.option_bits | TMP_TABLE_ALL_COLUMNS, > + HA_POS_ERROR /*rows_limit */, > + &order_nest_name))) > + return TRUE; /* purecov: inspected */ > + > + //tab->tmp_table_param= &order_nest_info->tmp_table_param; > + tab->type= JT_ALL; > + > + /* > + add_sort_to_table(); > + */ > + > + return false; > +} > + > static int > return_zero_rows(JOIN *join, select_result *result, List<TABLE_LIST> &tables, > List<Item> &fields, bool send_row, ulonglong select_options, > @@ -28550,6 +28821,26 @@ select_handler *SELECT_LEX::find_select_ > return 0; > } > > +double postjoin_oper_cost(THD *thd, double join_record_count, uint rec_len, uint idx) The name of the function is confusing. I see the function is modeled after spl_postjoin_oper_cost(), but that name is not readable either. Is this the cost of using a sort nest? > +{ > + double cost= 0; > + /* > + For only one table in the order_nest, we don't need a fill the temp table, we can > + just read the data into the filesort buffer and read the sorted data from the buffers. > + */ > + if (idx) Comparing idx with zero seems odd here. If we check whether this is the first table, we should compare with join->const_tables (This applies to spl_postjoin_oper_cost, too. Need to ask Igor about that one). > + cost= get_tmp_table_write_cost(thd, join_record_count,rec_len) * > + join_record_count; // cost to fill tmp table > + > + cost+= get_tmp_table_lookup_cost(thd, join_record_count,rec_len) * > + join_record_count; // cost to perform post join operation used here > + cost+= get_tmp_table_lookup_cost(thd, join_record_count, rec_len) + > + (join_record_count == 0 ? 0 : > + join_record_count * log2 (join_record_count)) * > + SORT_INDEX_CMP_COST; // cost to perform sorting > + return cost; > +} > + > > /** > @} (end of group Query_Optimizer) > diff -urpN '--exclude=.*' 10.5-mdev8306-orig/sql/sql_select.h 10.5-mdev8306/sql/sql_select.h > --- 10.5-mdev8306-orig/sql/sql_select.h 2019-07-10 11:34:00.260257044 +0300 > +++ 10.5-mdev8306/sql/sql_select.h 2019-07-10 11:34:15.448604784 +0300 > @@ -289,13 +289,14 @@ typedef struct st_join_table { > > /* TRUE <=> This join_tab is inside an SJM bush and is the last leaf tab here */ > bool last_leaf_in_bush; > - > + > /* > ptr - this is a bush, and ptr points to description of child join_tab > range > NULL - this join tab has no bush children > */ > JOIN_TAB_RANGE *bush_children; > + JOIN_TAB_RANGE *order_nest_children; This is not yet used anywhere? Is there any reason to have both bush_children and order_nest_children? I would suggest using bush_children pointer for both SJ-Materialization nests and sort nests. Another variable in the JOIN_TAB will be used to specify what kind of nest it is. (One JOIN_TAB cannot be both a SJ-Mat nest and a sort nest). > > /* Special content for EXPLAIN 'Extra' column or NULL if none */ > enum explain_extra_tag info; > @@ -524,6 +525,12 @@ typedef struct st_join_table { > /* Becomes true just after the used range filter has been built / filled */ > bool is_rowid_filter_built; > > + /* > + Set to true if we consider creating a nest for a prefix of the JOIN order > + that satisfies the ordering > + */ > + bool is_order_nest; This is a member of JOIN_TAB. If we get here, we are not considering, we've firmly decided to use a sort nest. I would prefer the name "sort nest" rather than order_nest. > + > void build_range_rowid_filter_if_needed(); > > void cleanup(); > @@ -991,6 +998,9 @@ typedef struct st_position > /* Cost info for the range filter used at this position */ > Range_rowid_filter_cost_info *range_rowid_filter_info; > > + /* Flag to be set to TRUE if the join prefix satisfies the ORDER BY CLAUSE */ > + bool ordering_achieved; As far as I understand, this is set to true if we have decided to add a sort_nest. We can add a sort nest when the join prefix covers the ORDER BY, but we don't have to. So, the name is confusing and should be changed to something like "sort_nest_operation_here" > + > } POSITION; > > typedef Bounds_checked_array<Item_null_result*> Item_null_array; BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

[Maria-developers] INSERT ... RETURNING Initial code review
by Vicențiu Ciorbaru 10 Jul '19

10 Jul '19

Hi Rucha! Here is the promised review. I have run the main test suite locally and have noticedthat some tests fail if you run with a debug build. The failure is an assertionfailure on the protocol level. This means you forgot to call a my_ok or my_errorat an appropriate time. I suggest you try and find which one of your commitsintroduces this bug. A handy tool is `git bisect`. It's a great time to learnabout it, if you have not used it before. There are quite a few other things to fix, but first, let's tackle theinitial review comments. Now, on to the actual code. Your text editor inserts tabs instead of spaces. Please configure it to onlyinsert tabs and have the "tab" size be equal to 2 spaces. > diff --git a/mysql-test/main/insert_parser.test b/mysql- test/main/insert_parser.test> new file mode 100644> index 00000000000..5c1f9408a96> --- /dev/null> +++ b/mysql- test/main/insert_parser.test> @@ -0,0 +1,64 @@> +#> +# Syntax check for INSERT...RETURNING> +#> +> +--echo # RETURNING * and RETURNING <COLUMN_NAME> should work. Trailing whitespace here, please fix.> + --echo # Other cases with RETURNING should output syntax error.> +> + --disable_warnings> +drop table if exists t1,t2;> + --enable_warningsTest cases in MariaDB are written such that they do not leave behind anytables or other elements from running the test. Thus, such statements arenot required (and actively discouraged) in new test cases. If tables fromprevious tests are present and mysql-test-run (mtr) does not mark the test asfailed then inserting such statements will hide the mysql-test-run bug.Please delete.> +> +CREATE TABLE t1(id1 INT, val1 VARCHAR(1));> +--echo Table t1 created successfully. Fields: id1 INT, val1 VARCHAR(1);This comment is not necessary, please delete. The fact that mtr passed meansthe command was succesful. Also, when adding comments, I recommend you prefixthem with #, like you did for the start of the test.> +> +CREATE TABLE t2(id2 INT, val2 VARCHAR(1));> +--echo Table t2 created successfully. Fields: id2 INT, val2 VARCHAR(1);Same as previous comment, not necessary, please delete.> +> +#> +#Simple insert statement> +#> +--echo #Simple insert statementOnly keep one, I suggest the --echo variant, but put a space after #.I also like to add 2 extra --echo # like so:--echo #--echo # Simple insert statement--echo # This makes the result file easier to read.> +INSERT INTO t1 (id1, val1) VALUES (1, 'a');> +INSERT INTO t1 (id1, val1) VALUES (2, 'b') RETURNING *;> +INSERT INTO t1 (id1, val1) VALUES (3, 'c') RETURNING id1, val1;> +INSERT INTO t1 (id1, val1) VALUES (3, 'c') RETURNING id1 AS id;> +SELECT * FROM t1;> +TRUNCATE TABLE t1;> +SELECT * FROM t1;> +> +#> +#multiple values in one insert statement> +#> +--echo #multiple values in one insert statementSame as above.> +INSERT INTO t1 VALUES (1,'a'),(2,'b');> +INSERT INTO t1 VALUES (3,'c'),(4,'d') RETURNING *;> +INSERT INTO t1 VALUES (5,'e'),(6,'f') RETURNING id1, val1;> +INSERT INTO t1 VALUES (5,'e'),(6,'f') RETURNING id1 AS id;> +> +#> +#INSERT...ON DULPICATE KEY UPDATE> +#> +--echo # INSERT...ON DULPICATE KEY UPDATESame as above. Typo too.> +CREATE TABLE ins_duplicate (id INT PRIMARY KEY, val VARCHAR(1));> +INSERT INTO ins_duplicate VALUES (1,'a');> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='b' RETURNING id,val;> +INSERT INTO ins_duplicate VALUES (3,'b') ON DUPLICATE KEY UPDATE val='c' RETURNING *;> +INSERT INTO ins_duplicate VALUES (4,'b') ON DUPLICATE KEY UPDATE val='d' RETURNING id AS id1;> +#> +# INSERT...SET> +#> +> +--echo # INSERT...SETSame as above.> +INSERT INTO t1 SET id1 = 1, val1 = 'a'; > +INSERT INTO t1 SET id1 = 2, val1 = 'b' RETURNING *;> +INSERT INTO t1 SET id1 = 3, val1 = 'c' RETURNING id1,val1;> +INSERT INTO t1 SET id1 = 3, val1 = 'c' RETURNING id1 AS id;> +> +DROP TABLE t1; > +DROP TABLE t2;> +DROP TABLE ins_duplicate;> +> +#> +--echo #End of test case > +#This is superfluous, please delete everything after the last DROP TABLE.> \ No newline at end of file> diff --git a/mysql- test/main/insert_parser_1.test b/mysql-test/main/insert_parser_1.test> new file mode 100644> index 00000000000..f83c288a800> --- /dev/null> +++ b/mysql-test/main/insert_parser_1.test> @@ -0,0 +1,75 @@Hmm, this file does not have a corresponding .result file. It's almost a copyof the previous one and it has windows line endings. It's the only one thathas those. I suggest you move the statements not present in insert_parser.testthere and delete this file entirely.> +#> +# Syntax check for INSERT...RETURNING> +#> +> +--disable_warnings> +drop table if exists t1,t2;> +--enable_warnings> +> +CREATE TABLE t1(id1 INT, val1 VARCHAR(1));> +--echo Table t1 created successfully. Fields: id1 INT, val1 VARCHAR(1);> +> +CREATE TABLE t2(id2 INT, val2 VARCHAR(1));> + --echo Table t2 created successfully. Fields: id2 INT, val2 VARCHAR(1);> +> +#> +#Simple insert statement> +#> +--echo #Simple insert statement> +INSERT INTO t1 (id1, val1) VALUES (1, 'a');> +INSERT INTO t1 (id1, val1) VALUES (2, 'b') RETURNING *;> +INSERT INTO t1 (id1, val1) VALUES (3, 'c') RETURNING id1, val1;> +INSERT INTO t1 (id1, val1) VALUES (4, 'd') RETURNING id1 as id;> +INSERT INTO t1 (id1, val1) VALUES (5, 'e') RETURNING id1 && id1;> +INSERT INTO t1 (id1, val1) VALUES (6, 'f') RETURNING id1 + id1;> +INSERT INTO t1 (id1, val1) VALUES (7, 'g') RETURNING id1 | id1;> +SELECT * FROM t1;> +TRUNCATE TABLE t1;> +SELECT * FROM t1;> +> +#> +#multiple values in one insert statement> +#> +--echo #multiple values in one insert statement> +INSERT INTO t1 VALUES (1,'a'),(2,'b');> +INSERT INTO t1 VALUES (3,'c'),(4,'d') RETURNING *;> +INSERT INTO t1 VALUES (5,'e'),(6,'f') RETURNING id1, val2;> +INSERT INTO t1 VALUES (7,'g'),(8,'h') RETURNING id1 AS id;> +INSERT INTO t1 VALUES (9,'i'),(10,'j') RETURNING id1 | id1 ;> +INSERT INTO t1 VALUES (11,'k'),(12,'l') RETURNING id1 + id1;> +INSERT INTO t1 VALUES (13,'m'),(14,'n') RETURNING id1 && id1;> +SELECT * FROM t1;> +> +#> +#INSERT...ON DULPICATE KEY UPDATE> +#> +--echo # INSERT...ON DULPICATE KEY UPDATE> +CREATE TABLE ins_duplicate (id INT PRIMARY KEY, val VARCHAR(1));> +INSERT INTO ins_duplicate VALUES (1,'a');> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='b' RETURNING id,val;> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='c' RETURNING *;> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='c' RETURNING id AS id1;> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='c' RETURNING id | id;> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='c' RETURNING id && id;> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='c' RETURNING id +id;> +SELECT * FROM ins_duplicate;> +> +#> +# INSERT...SET> +#> +--echo # INSERT...SET> +INSERT INTO t1 SET id1 = 1, val1 = 'a'; > +INSERT INTO t1 SET id1 = 2, val1 = 'b' RETURNING *;> +INSERT INTO t1 SET id1 = 3, val1 = 'c' RETURNING id1,val1;> +INSERT INTO t1 SET id1 = 3, val1 = 'c' RETURNING id1 as id;> +INSERT INTO t1 SET id1 = 3, val1 = 'c' RETURNING id1 | id1;> +INSERT INTO t1 SET id1 = 3, val1 = 'c' RETURNING id1 && id1;> +INSERT INTO t1 SET id1 = 3, val1 = 'c' RETURNING id1 + id1;> +> +DROP TABLE t1; > +DROP TABLE t2;> +DROP TABLE ins_duplicate;> +> +#> +--echo #End of test case > +#> \ No newline at end of file> diff --git a/mysql- test/main/insert_parser_sel.test b/mysql- test/main/insert_parser_sel.test> new file mode 100644> index 00000000000..f144cfc48ec> --- /dev/null> +++ b/mysql- test/main/insert_parser_sel.test> @@ -0,0 +1,45 @@> +#> +# Syntax check for INSERT...RETURNING> +#> +> +--disable_warnings> +drop table if exists t1,t2;> +--enable_warningsSimilar to insert_parser.test, these lines are not necessary.> +> +CREATE TABLE t1(id1 INT, val1 VARCHAR(1));> +--echo Table t1 created successfully. Fields: id1 INT, val1 VARCHAR(1);See insert_parser.test comments.> +> +CREATE TABLE t2(id2 INT, val2 VARCHAR(1));> +--echo Table t2 created successfully. Fields: id2 INT, val2 VARCHAR(1);See insert_parser.test comments.> +> +#> +#Simple insert statement> +#> +--echo #Simple insert statementSee insert_parser.test comments.> +INSERT INTO t1 (id1, val1) VALUES (1, 'a');> +INSERT INTO t1 (id1, val1) VALUES (2, 'b');> +INSERT INTO t1 (id1, val1) VALUES (3, 'c');> +INSERT INTO t1 (id1, val1) VALUES (4, 'd');> +INSERT INTO t1 (id1, val1) VALUES (5, 'e');> +INSERT INTO t1 (id1, val1) VALUES (6, 'f');> +INSERT INTO t1 (id1, val1) VALUES (7, 'g');> +SELECT * FROM t1;> +TRUNCATE TABLE t1;> +SELECT * FROM t1;> +> +#> +# INSERT...SELECT> +#> +--echo #INSERT...SELECTSee insert_parser.test comments.> +INSERT INTO t2(id2,val2) SELECT * FROM t1;> +INSERT INTO t2(id2,val2) SELECT * FROM t1 WHERE id1=1 RETURNING *;> +INSERT INTO t2(id2,val2) SELECT * FROM t1 WHERE id1=2 RETURNING id2,val2;> +INSERT INTO t2(id2,val2) SELECT * FROM t1 WHERE id1=3 RETURNING id2 AS id;> +INSERT INTO t2(id2,val2) SELECT * FROM t1 WHERE id1 IN (SELECT id1 FROM t1 WHERE id1 IN (3,4,5) );Ok, this last line does not test RETURNING at all, what was the intent?Also, please wrap queries to maximum 80 lines. Find the right way to indentthe SQL query on multiple lines to make it easier to read.> +SELECT * FROM t2;> +DROP TABLE t1; > +DROP TABLE t2;> +> +#> +--echo #End of test case > +#See insert_parser.test comments.> \ No newline at end of file> diff --git a/mysql-test/main/insert_returning_complex.test b/mysql- test/main/insert_returning_complex.test> new file mode 100644> index 00000000000..59ef3342e76> --- /dev/null> +++ b/mysql- test/main/insert_returning_complex.test> @@ -0,0 +1,117 @@> +#> +# Test for INSERT...RETURNING with virual cols,operators and # subqueries> +#> +> +--disable_warnings> +drop table if exists t1,t2,ins_duplicate;> + --enable_warningsSee insert_returning.testI would merge this test into the insert_returning.test file. No need to split them up.> +> +CREATE TABLE t1(id1t1 INT, id2t1 INT, valt1 VARCHAR(1));> +CREATE TABLE t2(id1t2 INT, id2t2 INT, valt2 VARCHAR(1));> +INSERT INTO t2 VALUES (1,2,'a'),(2,3,'b'),(3,4,'c'),(4,5,'d'),(5,6,'e'),(6,7,'f'),(7,8,'g'),( 8,9,'h'),(9,10,'i'),(10,11,'j');Please wrap this line to 80 characters max.> +> +--echo Table t1 created successfully. Fields: id1 INT, id2 INT, val VARCHAR(1);Superfluous.> +> +#> +#Simple insert statement> +#> +--echo #Simple insert statementSee insert_returning.test comments.> +INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (1, 2, 'a');> +INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (2, 3, 'b') RETURNING *;> +INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (3, 4, 'c') RETURNING id1t1, valt1;> +INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (4, 5, 'd') RETURNING id1t1 AS id;> +INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (5, 6, 'e') RETURNING id1t1+id2t1 AS total;> +INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (6, 7, 'f') RETURNING id1t1 & id2t1;> +INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (7, 8, 'g') RETURNING id1t1 || id2t1;> +INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (8,9,'h') RETURNING id1t1,UPPER(valt1);> +INSERT INTO t1(id1t1,id2t1,valt1) VALUES (9,10,'i') RETURNING (SELECT GROUP_CONCAT(valt2) FROM t2 WHERE id1t2=1);> +INSERT INTO t1(id1t1,id2t1,valt1) VALUES(10,11,'j') RETURNING (SELECT GROUP_CONCAT(valt2) FROM t2 GROUP BY id1t2 HAVING id1t2=id1t2+1);Ok, please wrap all such lines to 80 characters to make them easier to parse.Second, can we test with subqueries that return multiple rows, for example:SELECT GROUP_CONCAT(valt2) FROM t2 group by id1t2.We should return an error here, but this should be covered in the test casetoo.> +INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (8, 9, 'h') RETURNING id1t1,(SELECT id2t2 FROM t2 WHERE valt2='b');Can you also test a subquery that uses columns from t1?Example:INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (8, 9, 'h') RETURNING id1t1, (SELECT id2t2 + id1t1 FROM t2 WHERE valt2='b');> > +SELECT * FROM t1;> +TRUNCATE TABLE t1;> +SELECT * FROM t1;> +> +> +#> +#multiple values in one insert statement> +#> +--echo #multiple values in one insert statementSee insert_returning.test comments.> +INSERT INTO t1 VALUES (1, 2, 'a'),(2, 3, 'b');> +INSERT INTO t1 VALUES (3, 4, 'c'),(4, 5, 'd') RETURNING *;> +INSERT INTO t1 VALUES (5, 6, 'e'),(6, 7, 'f') RETURNING id2t1, valt1;> +INSERT INTO t1 VALUES (7, 8, 'g'),(8, 9, 'h') RETURNING id2t1 AS id;> +INSERT INTO t1 VALUES (9, 10, 'i'),(10, 11, 'j') RETURNING id2t1+id1t1 as total;> +INSERT INTO t1 VALUES (11, 12, 'k'),(12, 13, 'l') RETURNING id2t1 & id1t1;> +INSERT INTO t1 VALUES (5, 6, 'e'),(2, 3, 'f') RETURNING id2t1 || id1t1;> +INSERT INTO t1 VALUES (13,14,'m'),(14,15,'n') RETURNING id1t1,UPPER(valt1);> +INSERT INTO t1 VALUES (15,16,'o'),(16,17,'p') RETURNING (SELECT GROUP_CONCAT(valt2) FROM t2 WHERE id1t2=1);> +INSERT INTO t1 VALUES (17,18,'q'),(18,19,'r') RETURNING (SELECT GROUP_CONCAT(valt2) FROM t2 GROUP BY id1t2 HAVING id1t2=id1t2+1)> +INSERT INTO t1 VALUES (19,20,'r'),(20,21,'s') RETURNING id1t1,(SELECT id2t2 FROM t2 WHERE valt2='b');Same comments as for single-value insert.> +SELECT * FROM t1;> +TRUNCATE TABLE t1;> +> +#> +#INSERT...ON DULPICATE KEY UPDATE> +#> +--echo # INSERT...ON DULPICATE KEY UPDATETypo DULPICATE -> DUPLICATE. Also, see comments from insert_returning.test.> +CREATE TABLE ins_duplicate (id1 INT PRIMARY KEY, id2 INT, val VARCHAR(1));> +INSERT INTO ins_duplicate VALUES (1,2,'a');> +INSERT INTO ins_duplicate VALUES (2,3,'b') ON DUPLICATE KEY UPDATE val='b' RETURNING id1,val;> +INSERT INTO ins_duplicate VALUES (2,4,'b') ON DUPLICATE KEY UPDATE val='c' RETURNING *;> +INSERT INTO ins_duplicate VALUES (2,5,'b') ON DUPLICATE KEY UPDATE val='d' RETURNING id1+id2 AS total;> +INSERT INTO ins_duplicate VALUES (2,6,'b') ON DUPLICATE KEY UPDATE val='e' RETURNING id1 || id2;> +INSERT INTO ins_duplicate VALUES (2,7,'b') ON DUPLICATE KEY UPDATE val='f' RETURNING id1 & id2;> +INSERT INTO ins_duplicate VALUES (2,8,'b') ON DUPLICATE KEY UPDATE val='g' RETURNING id1 AS id;> +INSERT INTO ins_duplicate VALUES (2,9,'b') ON DUPLICATE KEY UPDATE val='h' RETURNING id1,UPPER(val);> +INSERT INTO ins_duplicate VALUES (2,10,'b') ON DUPLICATE KEY UPDATE val='i' RETURNING (SELECT GROUP_CONCAT(valt2) FROM t2 WHERE id1t2=1);> +INSERT INTO ins_duplicate VALUES (2,11,'b') ON DUPLICATE KEY UPDATE val='j' RETURNING (SELECT GROUP_CONCAT(valt2) FROM t2 GROUP BY id1t2 HAVING id1t2=id1t2+1);> +INSERT INTO ins_duplicate VALUES (2,12,'b') ON DUPLICATE KEY UPDATE val='k' RETURNING id1,(SELECT id2t2 FROM t2 WHERE valt2='b');> +SELECT * FROM ins_duplicate;> +> +#> +# INSERT...SET> +#> +> +--echo # INSERT...SET> +INSERT INTO t1 SET id1t1=1, id2t1=2, valt1='a'; > +INSERT INTO t1 SET id1t1=2, id2t1=3, valt1='b' RETURNING *;> +INSERT INTO t1 SET id1t1=3, id2t1=4, valt1='c' RETURNING valt1;> +INSERT INTO t1 SET id1t1=4, id2t1=5, valt1='d' RETURNING id1t1+id2t1 AS total;> +INSERT INTO t1 SET id1t1=5, id2t1=6, valt1='e' RETURNING id1t1 & id2t1;> +INSERT INTO t1 SET id1t1=6, id2t1=7, valt1='f' RETURNING id1t1 || id2t1;> +INSERT INTO t1 SET id1t1=7, id2t1=8, valt1='g' RETURNING valt1 AS letter;> +INSERT INTO t1 SET id1t1=8, id2t1=9, valt1='h' RETURNING id1t1,UPPER(valt1);> +INSERT INTO t1 SET id1t1=9, id2t1=10, valt1='i' RETURNING (SELECT GROUP_CONCAT(valt2) FROM t2 WHERE id1t2=1);> +INSERT INTO t1 SET id1t1=10, id2t1=11, valt1='j' RETURNING (SELECT GROUP_CONCAT(valt2) FROM t2 GROUP BY id1t2 HAVING id1t2=id1t2+1);> +INSERT INTO t1 SET id1t1=11, id2t1=12, valt1='k' RETURNING id1t1,(SELECT id2t2 FROM t2 WHERE valt2='b'); > +SELECT * FROM t1;> +TRUNCATE TABLE t1;> +SELECT * FROM t1;> +> +#> +# INSERT...SELECT> +#> +--echo # INSERT...SELECT> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=1;> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=2 RETURNING *;> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=3 RETURNING valt1;> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=4 RETURNING id1t1+id2t1 AS total;> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=5 RETURNING id1t1 & id2t1;> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=6 RETURNING valt1 AS letter;> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=2 RETURNING id1t1 || id2t1;> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=7 RETURNING id1t1,UPPER(valt1);> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=8> +RETURNING (SELECT GROUP_CONCAT(val) FROM ins_duplicate WHERE id1=1);> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=9> +RETURNING (SELECT GROUP_CONCAT(val) FROM ins_duplicate GROUP BY id1 HAVING id1=id1+1);> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=10 RETURNING id1t1,(SELECT id2 FROM ins_duplicate WHERE valt2='b');> +SELECT* FROM t1;> +> +--echo Droping t1 and ins_duplicate> +DROP TABLE t1;> +DROP TABLE t2; > +DROP TABLE ins_duplicate;> +> +#> +--echo #End of test case > +# > \ No newline at end of file> Binary files /dev/null and b/mysql- test/main/insert_returning_datatypes.result differOk, why is this a binary file for git? Something went wrong here.> diff --git a/mysql- test/main/insert_returning_datatypes.test b/mysql- test/main/insert_returning_datatypes.test> new file mode 100644> index 00000000000..803c8326b6f> --- /dev/null> +++ b/mysql- test/main/insert_returning_datatypes.test> @@ -0,0 +1,92 @@> +#> +# Tests for DELETE FROM <table> ... RETURNING <expr>,...> +# What is DELETE FROM .. RETURNING doing here? :)> +> +--disable_warnings> +drop table if exists t1,t2;> +--enable_warningsI vote for merging this into the previous tests.> +> +CREATE TABLE t1(num_int1 INT(2) PRIMARY KEY, > + num_bit1 BIT(2),> + num_float1 FLOAT(5,2),> + num_double1 DOUBLE(5,2),> + char_enum1 ENUM('A','B','C','D'),> + char_set1 SET('a','b','c','d','e'),> + str_varchar1 VARCHAR(2),> + str_binary1 BINARY(2),> + d1 DATE,> + dt1 DATETIME,> + ts1 TIMESTAMP,> + y1 YEAR,> + b1 BOOL);> +> +CREATE TABLE t2(num_int2 INT(2) PRIMARY KEY, > + num_bit2 BIT(2),> + num_float2 FLOAT(5,2),> + num_double2 DOUBLE(5,2),> + char_enum2 ENUM('A','B','C','D'),> + char_set2 SET('a','b','c','d','e'),> + str_varchar2 VARCHAR(2),> + str_binary2 BINARY(2),> + d2 DATE,> + dt2 DATETIME,> + ts2 TIMESTAMP,> + y2 YEAR,> + b2 BOOL);> +> +> +#> +# SIMLPE INSERT STATEMENT> +#Typo, but please delete this anayway.> +--echo # SIMPLE INSERT STATEMENTSee other test comments.> +INSERT INTO t1(num_int1,num_bit1,num_float1,num_double1,char_enum1,char_set1,str_va rchar1, str_binary1,d1,dt1,ts1,y1,b1) VALUES(1,0, 123.45, 123.55, 'A','b,e', 'V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,0) RETURNING *;> +> +INSERT INTO t1(num_int1,num_bit1,num_float1,num_double1,char_enum1,char_set1,str_va rchar1, str_binary1,d1,dt1,ts1,y1,b1) VALUES(2,0, 123.45, 123.55, 'A','b,e', 'V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,0) RETURNING num_int1,num_bit1,d1,dt1;> +> +> +> +#> +# MULTIPLE ROWS IN SINGLE STATEMENT > +#> +--echo #Multiple rows inserted in one statement> +INSERT INTO t1 VALUES(3,0, 123.45, 123.55, 'A','b,e', 'V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,0),(4,0, 123.45, 123.55, 'A','b,e', 'V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,1),(5,0, 123.45, 123.55, 'A','b,e','V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,1) RETURNING *;> +> +INSERT INTO t1 VALUES(6,0, 123.45, 123.55, 'A','b,e', 'V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,0),(7,0, 123.45, 123.55, 'A','b,e', 'V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,1),(8,0, 123.45, 123.55, 'A','b,e','V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,1) RETURNING char_set1,ts1,y1;> +> +> +> +#> +#INSERT...SET...RETURNING> +#> +--echo #INSERT...SET...RETURNING> +INSERT INTO t1 SET num_int1=9,num_bit1=0,num_float1=124.67,num_double1=231.12,char_enum1=' B',char_set1='a,d,e',str_varchar1='AB',str_binary1='01', d1='120314',dt1="2012-04-19 13:08:22",ts1='2001-07-22 12:12:1',y1=2014,b1=1 RETURNING *;> +> +INSERT INTO t1 SET num_int1=10,num_bit1=0,num_float1=124.67,num_double1=231.12,char_enum1= 'B',char_set1='a,d,e',str_varchar1='AB',str_binary1='01',d1='120314',dt 1="2012-04-19 13:08:22",ts1='2001-07-22 12:12:1',y1=2014,b1=1 RETURNING b1,char_enum1;> +> +> +> +#> +# INSERT...ON DUPLICATE KEY UPDATE> +# > +--echo # INSERT...ON DUPLCATE KEY UPDATE...RETURNING> +INSERT INTO t1(num_int1,num_bit1,num_float1,num_double1,char_enum1,char_set1,str_va rchar1, str_binary1,d1,dt1,ts1,y1,b1) VALUES (10,0, 123.45, 123.55, 'C','b,e', 'V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,0) ON DUPLICATE KEY UPDATE num_float1=111.111 RETURNING *;> +> +INSERT INTO t1(num_int1,num_bit1,num_float1,num_double1,char_enum1,char_set1,str_va rchar1, str_binary1,d1,dt1,ts1,y1,b1) VALUES (11,0, 123.45, 123.55, 'A','b,e', 'V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,0) ON DUPLICATE KEY UPDATE char_set1='a,b' RETURNING char_set1,num_int1;> +> +> +> +#> +# INSERT...SELECT...RETURNING> +#> +--echo # INSERT...SELECT...RETURNING> +INSERT INTO t2 (num_int2,num_bit2,num_float2,num_double2,char_enum2,char_set2,str_varc har2, str_binary2,d2,dt2,ts2,y2,b2) SELECT * FROM t1 RETURNING *;> +TRUNCATE TABLE t2;> +INSERT INTO t2 (num_int2,num_bit2,num_float2,num_double2,char_enum2,char_set2,str_varc har2, str_binary2,d2,dt2,ts2,y2,b2) SELECT * FROM t1 RETURNING num_double2,str_varchar2,str_binary2;> +> +> +DROP TABLE t1;> +DROP TABLE t2;> +> +#> +#END OF TEST CASE> +# > \ No newline at end of file> diff --git a/mysql-test/main/insert_returning_err.test b/mysql- test/main/insert_returning_err.test> new file mode 100644> index 00000000000..4af798e5c68> --- /dev/null> +++ b/mysql- test/main/insert_returning_err.test> @@ -0,0 +1,97 @@> +#> +# Test for checking error message for INSERT...RETURNING> +#> +> +#INSERT INTO <table> ... RETURNING <not existing col>> +#INSERT INTO <table> ... RETURNING <expr with aggr function>> +#INSERT INTO ... RETURNING subquery with more than 1 row> +#INSERT INTO ... RETURNING operand should contain 1 colunm(s)> +#INSERT INTO ... RETURNING operand should contain 1 colunm(s)> +> +--disable_warnings> +drop table if exists t1,t2;> +--enable_warnings> +> +CREATE TABLE t1(id1 INT,val1 VARCHAR(1));> +CREATE TABLE t2(id2 INT,val2 VARCHAR(1));> +CREATE TABLE ins_duplicate (id INT PRIMARY KEY, val VARCHAR(1));> +> +INSERT INTO t1 VALUES(1,'a'),(2,'b'),(3,'c');> +> +#> +# SIMLPE INSERT STATEMENT> +#> +--error ER_BAD_FIELD_ERROR> +INSERT INTO t2(id2,val2) VALUES(1,'a') RETURNING id1;> +--error ER_INVALID_GROUP_FUNC_USE> +INSERT INTO t2(id2,val2) values(2,'b') RETURNING MAX(id2);> +--error ER_SUBQUERY_NO_1_ROW> +INSERT INTO t2(id2,val2) VALUES(3,'c') RETURNING (SELECT id1 FROM t1);Ok, so you do test this, good. What about a subquery using a column from t2?> +--error ER_OPERAND_COLUMNS> +INSERT INTO t2(id2,val2) VALUES(4,'d') RETURNING (SELECT * FROM t1);This query fails specifically because there are multiple columns in the subselect.What happens if there is only one column?> +--error ER_OPERAND_COLUMNS> +INSERT INTO t2(id2,val2) VALUES(4,'d') RETURNING (SELECT * FROM t2);> +> +#> +# Multiple rows in single insert statement> +#> +--error ER_BAD_FIELD_ERROR> +INSERT INTO t2 VALUES(1,'a'),(2,'b') RETURNING id1;> +--error ER_INVALID_GROUP_FUNC_USE> +INSERT INTO t2 VALUES(3,'c'),(4,'d') RETURNING MAX(id2);> +--error ER_SUBQUERY_NO_1_ROW> +INSERT INTO t2 VALUES(5,'c'),(6,'f') RETURNING (SELECT id1 FROM t1);> +--error ER_OPERAND_COLUMNS> +INSERT INTO t2 VALUES(7,'g'),(8,'h') RETURNING (SELECT * FROM t1);> +--error ER_OPERAND_COLUMNS> +INSERT INTO t2 VALUES(7,'g'),(8,'h') RETURNING (SELECT * FROM t2);> +> +#> +# INSERT ... SET> +#> +--error ER_BAD_FIELD_ERROR> +INSERT INTO t2 SET id2=1, val2='a' RETURNING id1;> +--error ER_INVALID_GROUP_FUNC_USE> +INSERT INTO t2 SET id2=2, val2='b' RETURNING MAX(id2);> +--error ER_SUBQUERY_NO_1_ROW> +INSERT INTO t2 SET id2=3, val2='c' RETURNING (SELECT id1 FROM t1);> +--error ER_OPERAND_COLUMNS> +INSERT INTO t2 SET id2=4, val2='d' RETURNING (SELECT * FROM t1);> +--error ER_OPERAND_COLUMNS> +INSERT INTO t2 SET id2=4, val2='d' RETURNING (SELECT * FROM t2);> +> +#> +#INSERT...ON DUPLICATE KEY UPDATE> +#> + --error ER_BAD_FIELD_ERROR> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='b' RETURNING id1;> +--error ER_INVALID_GROUP_FUNC_USE> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='b' RETURNING MAX(id);> +--error ER_SUBQUERY_NO_1_ROW> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='b' RETURNING (SELECT id1 FROM t1);> +--error ER_OPERAND_COLUMNS> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='b' RETURNING (SELECT * FROM t1);> +--error ER_OPERAND_COLUMNS> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='b' RETURNING (SELECT * FROM ins_duplicate);> +> +#> +#INSERT...SELECT...RETURNING> +#Use --echo # for the lines above please.> +--error ER_BAD_FIELD_ERROR> +INSERT INTO t2(id2, val2) SELECT * FROM t1 WHERE id1=1 RETURNING id1;> +--error ER_INVALID_GROUP_FUNC_USE> +INSERT INTO t2(id2, val2) SELECT * FROM t1 WHERE id1=2 RETURNING MAX(id2);> +--error ER_SUBQUERY_NO_1_ROW> +INSERT INTO t2(id2, val2) SELECT * FROM t1 WHERE id1=2 RETURNING (SELECT id1 FROM t1);> +--error ER_OPERAND_COLUMNS> +INSERT INTO t2(id2, val2) SELECT * FROM t1 WHERE id1=2 RETURNING (SELECT * FROM t1);> +--error ER_OPERAND_COLUMNS> +INSERT INTO t2(id2, val2) SELECT * FROM t1 WHERE id1=2 RETURNING (SELECT * FROM t2);> +> +DROP TABLE t1;> +DROP TABLE t2;> +DROP TABLE ins_duplicate;> +> +#> +# End of test case> +# > \ No newline at end of file> diff --git a/mysql- test/main/insert_returning_stmt.result b/mysql- test/main/insert_returning_stmt.result> new file mode 100644> index 00000000000..3b48cb11b28> --- /dev/null> +++ b/mysql- test/main/insert_returning_stmt.resultHow is this test different than previous ones? I am getting a bit confusedby the large number of test files that seem simillar.> \ No newline at end of file> diff --git a/mysql-test/main/insert_sel_returning.test b/mysql- test/main/insert_sel_returning.test> new file mode 100644> index 00000000000..8b4880e2064> --- /dev/null> +++ b/mysql- test/main/insert_sel_returning.test> diff --git a/sql/sql_class.h b/sql/sql_class.h> index 18ccf992d1e..8e67818f891 100644> --- a/sql/sql_class.h> +++ b/sql/sql_class.h> @@ -5475,13 +5475,15 @@ class select_dump :public select_to_file {> > class select_insert :public select_result_interceptor {> public:> + select_result* sel_result;> + bool with_returning_list;> TABLE_LIST *table_list;> TABLE *table;> List<Item> *fields;> ulonglong autoinc_value_of_last_inserted_row; // autogenerated or not> COPY_INFO info;> bool insert_into_view;> - select_insert(THD *thd_arg, TABLE_LIST *table_list_par,> + select_insert(bool with_ret_list, select_result* sel_ret_list, THD *thd_arg, TABLE_LIST *table_list_par,When adding parameters, please add them to the end of the function, not thestart. Do we need both with_returning_list and sel_result pointer? Can wenot use just the sel_result pointer and if it is null, we assume we do nothave a returning list?> TABLE *table_par, List<Item> *fields_par,> List<Item> *update_fields, List<Item> *update_values,> enum_duplicates duplic, bool ignore);> diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc> index f3a548d7265..6eb9883fe87 100644> --- a/sql/sql_insert.cc> +++ b/sql/sql_insert.cc> @@ -1236,26 +1261,38 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list,> if ((iteration * values_list.elements) == 1 && (!(thd->variables.option_bits & OPTION_WARNINGS) ||> !thd->cuted_fields))> {> - my_ok(thd, info.copied + info.deleted +> - ((thd- >client_capabilities & CLIENT_FOUND_ROWS) ?> - info.touched : info.updated),> - id);> + if (with_returning_list)> + result->send_eof();> + else > + {> + my_ok(thd, info.copied + info.deleted +> + ((thd->client_capabilities & CLIENT_FOUND_ROWS) ?> + info.touched : info.updated),> + id);> + }> }> e lse> {> char buff[160];> ha_rows updated=((thd- >client_capabilities & CLIENT_FOUND_ROWS) ?> info.touched : info.updated);> - if (ignore)> - sprintf(buff, ER_THD(thd, ER_INSERT_INFO), (ulong) info.records,> - (lock_type == TL_WRITE_DELAYED) ? (ulong) 0 :> - (ulong) (info.records - info.copied),> - (long) thd->get_stmt_da()- >current_statement_warn_count());> - else> - sprintf(buff, ER_THD(thd, ER_INSERT_INFO), (ulong) info.records,> - (ulong) (info.deleted + updated),> - (long) thd->get_stmt_da()- >current_statement_warn_count());> - ::my_ok(thd, info.copied + info.deleted + updated, id, buff);> + if (ignore)> + sprintf (buff, ER_THD(thd, ER_INSERT_INFO), (ulong)info.records,> + (lock_type == TL_WRITE_DELAYED) ? (ulong)0 :> + (ulong) (info.records - info.copied),> + (long)thd- >get_stmt_da()->current_statement_warn_count());> + else > + {> + if (with_returning_list)> + result- >send_eof();> + else > + {> + sprintf(buff, ER_THD(thd, ER_INSERT_INFO), (ulong)info.records,> + (ulong)(info.deleted + updated),> + (long)thd->get_stmt_da()- >current_statement_warn_count());> + ::my_ok(thd, info.copied + info.deleted + updated, id, buff);> + }> + }Uhm, careful here, you have changed the logic of when my_ok gets called! Please re-read this bit of code and try to understand where things havechanged. Also, looking at this, can you test INSERT IGNORE ... RETURNING too? I have a feeling it won't work with the current implementation.> }> thd->abort_on_warning= 0;> if (thd->lex- >current_select->first_cond_optimization)> @@ -1557,7 +1595,10 @@ bool mysql_prepare_insert(THD *thd, TABLE_LIST *table_list,> table_list->next_local= 0;> context- >resolve_in_table_list_only(table_list);> > - res= (setup_fields(thd, Ref_ptr_array(),> + res= ((with_returning_list?((sel_lex->with_wild && setup_wild(thd, table_list, sel_lex->returning_list, NULL, sel_lex->with_wild,> + &select_lex->hidden_bit_fields)) ||> + setup_fields(th d, Ref_ptr_array(),> + sel_lex->returning_list, MARK_COLUMNS_READ, 0, NULL, 0)):false)||setup_fields(thd, Ref_ptr_array(),Uhm, please fix the formatting for this code. The logic is also quite convoluted,can you turn this into something actually readable? Also, you are calling the same setup_wild && setup_fields call both withinmysql_prepare_insert, but also within select_result->prepare. Why 2 times?> *values, MARK_COLUMNS_READ, 0, NULL, 0) ||> check_insert_fields(thd, context->table_list, fields, *values,> !insert_into_view, 0, &map));> @@ -3557,12 +3598,13 @@ bool Delayed_insert::handle_inserts(void)> TRUE Error> */> > -bool mysql_insert_select_prepare(THD *thd)> +bool mysql_insert_select_prepare(THD *thd,select_result *sel_res)> {> LEX *lex= thd->lex;> SELECT_LEX *select_lex= lex- >first_select_lex();> DBUG_ENTER("mysql_insert_select_prepare");> -> + bool with_ret = lex->current_select- >returning_list.is_empty()?false:true;> + List<Item>& returning_list = thd->lex->current_select->returning_list;Not really a fan of this method. By doing this, it's going to be impossibleto extend INSERT .. RETURNING to work within subqueries.> > /*> SELECT_LEX do not belong to INSERT statement, so we can't add WHERE> @@ -3572,14 +3614,18 @@ bool mysql_insert_select_prepare(THD *thd)> if (mysql_prepare_insert(thd, lex- >query_tables,> lex->query_tables->table, lex->field_list, 0,> lex->update_list, lex- >value_list, lex->duplicates,> - &select_lex- >where, TRUE))> + &select_lex->where, TRUE,with_ret?select_lex:NULL))I don't like that we end up passing select_lex to mysql_prepare_insert.Can we make the arguments of this function be strictly what is necessary,not the full SELECT_LEX structure?> DBUG_RETURN(TRUE);> > + if (with_ret)> + (void )sel_res->prepare(returning_list, NULL);> +> DBUG_ASSERT(select_lex- >leaf_tables.elements != 0);> List_iterator<TABLE_LIST> ti(select_lex->leaf_tables);> TABLE_LIST *table;> uint insert_tables;> > +> if (select_lex- >first_cond_optimization)> {> /* Back up leaf_tables list. */> @@ -3630,6 +3677,8 @@ select_insert::select_insert(THD *thd_arg, TABLE_LIST *table_list_par,> info.update_values= update_values;> info.view= (table_list_par->view ? table_list_par : 0);> info.table_list= table_list_par;> + sel_result= result;> + with_returning_list= with_ret_list;Can we get rid of with_returning_list and just use sel_result? If it's null,with_returning_list should also be null.> }> > > @@ -3651,7 +3701,28 @@ select_insert::prepare(List<Item> &values, SELECT_LEX_UNIT *u)> */> lex->current_select= lex->first_select_lex();> > - res= (setup_fields(thd, Ref_ptr_array(),> + /*> + We want the returning_list to point to insert table. But the context is masked. > + So we swap it with the context saved during parsing stage. > + */> + if(with_returning_list) > + { swap_context(select_lex- >table_list.saved_first,select_lex->table_list.first);> + swap_context(select_lex->context.saved_table_list,select_lex- >context.table_list);> + swap_context(select_lex- >context.saved_name_resolution_table,select_lex- >context.first_name_resolution_table);> +> + res=((select_lex- >with_wild && setup_wild(thd, table_list, select_lex->returning_list, NULL, select_lex->with_wild,> + &select_lex- >hidden_bit_fields)) ||> + setup_fields(thd, Ref_ptr_array(),> + select_lex->returning_list, MARK_COLUMNS_READ, 0, NULL, 0));> +> + /*Swap it back to retore the previous state for the rest of the function*/> +> + swap_context(select_lex->table_list.saved_first,select_lex- >table_list.first);> + swap_context(select_lex- >context.saved_table_list,select_lex->context.table_list);> + swap_context(select_lex->context.saved_name_resolution_table, select_lex->context.first_name_resolution_table);> + }I am not sure this will work if you have columns with the same names, bothin t1 and t2 (your tests don't cover this).Ex:CREATE table t1 (pk int);CREATE table t2 (pk int);INSERT INTO t2 (pk) values (1), (2); INSERT INTO t1 SELECT * FROM t2 RETURNING pk; Also, what about INSERT INTO t1 SELECT .. FROM t1? Please add a test casefor this and see if the code still continues to work. Also, the saved_first introduction into SQL_I_List is really a hack.One needs to save the correct list of tables separately, properly. Pleasefind a solution without changing SQL_I_List. If this part gets you stuck,we should discuss it, but I want you first to try and find a proper solutionfirst.> +> + res= res || (setup_fields(thd, Ref_ptr_array(),> values, MARK_COLUMNS_READ, 0, NULL, 0) ||> check_insert_fields(thd, table_list, *fields, values,> !insert_into_view, 1, &map));> @@ -4046,7 +4136,10 @@ bool select_insert::send_ok_packet() {> thd- >first_successful_insert_id_in_prev_stmt :> (info.copied ? autoinc_value_of_last_inserted_row : 0));> > - ::my_ok(thd, row_count, id, message);> + if (with_returning_list)> + sel_r esult->send_eof();> + else> + ::my_ok(thd, row_count, id, message);I believe you need to return my_ok regardless.> > DBUG_RETURN(false);> }> diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc> index 01d0ed1c383..6cee252096b 100644> --- a/sql/sql_parse.cc> +++ b/sql/sql_parse.cc> @@ -4604,11 +4664,21 @@ mysql_execute_command(THD *thd)> TODO: fix it by removing the front element (restoring of it should> be done properly as well)> */> + /*> + If items are present in returning_list, then we need those items to point to > + INSERT table during setup_fields() and setup_wild(). But it gets masked before that. > + So we save the values in saved_first, saved_table_list and saved_first_name_resolution_context.> + before they are masked.> + */> + select_lex- >table_list.saved_first=select_lex->table_list.first;Right, this is a hack. Please check my other comment about adding saved_first.Please find a way to get the correct list of tables for name resolutionwithout extending SQL_I_List.> select_lex->table_list.first= second_table;> + select_lex- >context.saved_table_list=select_lex->context.table_list;> + selec t_lex->context.saved_name_resolution_table=> + selec t_lex->context.first_name_resolution_table;> select_lex- >context.table_list= > select_lex- >context.first_name_resolution_table= second_table;> - res= mysql_insert_select_prepare(thd);> - if (!res && (sel_result= new (thd->mem_root) select_insert(thd,> + res= mysql_insert_select_prepare(thd, lex->result ? lex->result : result);> + if (!res && (sel_result= new (thd->mem_root) select_insert(with_returning_list, lex->result ? lex->result : result, thd,> fir st_table,> first_table- >table,> &lex->field_list,> diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy> index 183a2504b70..3ef53350101 100644> --- a/sql/sql_yacc.yy> +++ b/sql/sql_yacc.yy> @@ -13333,10 +13338,15 @@ replace:> }> insert_field_spec> {> - Lex->pop_select(); //main select> - if (Lex- >check_main_unit_semantics())> - MYSQL_YYABORT;> + Lex->current_select->returning_list.swap(Lex- >current_select->item_list);> }> + opt_select_ex pressions> + {> + Lex->current_select- >returning_list.swap(Lex->current_select->item_list);> + Lex->pop_select(); //main select> + if (Lex- >check_main_unit_semantics())> + MYSQL_YYABORT;> + }Please fix indentation.> ;> > insert_lock_option:

2 2

[Maria-developers] CONC/C and CONC/ODBC API and ABI compatibility
by Michal Schorm 08 Jul '19

08 Jul '19

Hello, I am a Fedora packager and I'd like to make myself sure on some assumptions I made. Just to make sure in the following conversation: * I understand "Connector C 3.1.2" as "major version number . minor version number . release number (or sometimes refered as patch number)". * In Fedora we use dynamic libraries and dynamic linking whenever possible. We are building the CONC/ODBC on top of CONC/C. 1) ABI & API compatibility I assume, the major versions are fully API & ABI (backward) compatible. That means to me it is safe to upgrade to later versions of the same major version number when using end-user application. There's no need to rebuild the end-user app. "The MariaDB Connector/ODBC 3.1 series is built on top of MariaDB Connector/C 3.x" That means to me I can build the CONC/ODBC 3.1 on top of CONC/C 3.0 and there's no need for rebuild, when I upgrade to the CONC/C 3.1. Thus it is safe to upgrade withing one major version without any threat. That means to me I can update the package in Fedora mid release. During a lifetime of a Fedora release (let's say Fedora 30), I only update packages which won't break ABI & API compatibility. If there would be potential risk, I would push that update only to the next Fedora release (developement branch - Fedora Rawhide). That's what I do e.g. with minor versions of MariaDB server. I won't update the 10.3 to 10.4 in the lifetime of one release, but prepare the 10.4 to the next release. (Fedora Rawhide) Am I right with above assumptions? Can I safely upgrade within one major version of the C and ODBC connectors and don't need rebuild of the dependant packages and end-user apps? If not, why was CONC/ODBC 3.0 discontinued and hidden? 2) Version upgrade Is there any reason to use other than latest GA release of those connectors? (e.g. to use CONC/C 3.0 series instead of 3.1 series) Perhaps when using against different MariaDB server major versions? I assume there is not. If there is, again - why was CONC/ODBC 3.0 discontinued and hidden? -- Michal Schorm Software Engineer Core Services - Databases Team Red Hat --

3 4

[Maria-developers] Weekly Report
by Rucha Deodhar 08 Jul '19

08 Jul '19

Report for week 6: Hello! This week I was able to make replace...returning work. But before pushing changes for insert...select...returning, I had done a git merge because I didn't know I was supposed to do a git rebase. While undoing the merge I ran into a problem. With help from mentors I was able to fix the problem. I had to make some commits again. It took me two days to fix the issue and make commits and then I was on track again. I also made a GDoc about using insert...returning in subqueries. I have received feedback about my code and I have started working on fixing it. Regards, Rucha Deodhar <https://www.avast.com/en-in/recommend?utm_medium=email&utm_source=link&utm_…> I’m protected online with Avast Free Antivirus. Get it here — it’s free forever. <https://www.avast.com/en-in/recommend?utm_medium=email&utm_source=link&utm_…> <#DAB4FAD8-2DD7-40BB-A1B8-4E2AA1F9FDF2>

1 0

Re: [Maria-developers] 4670c85a48c: MDEV-16222 Assertion `0' failed in row_purge_remove_sec_if_poss_leaf on table with virtual columns and indexes
by Sergei Golubchik 05 Jul '19

05 Jul '19

Hi, Aleksey! Summary: looks ok, but please move clear_error() to the caller (ha_innodb.cc) and remove the assert. On Jul 04, Aleksey Midenkov wrote: > revision-id: 4670c85a48c (mariadb-10.4.5-47-g4670c85a48c) > parent(s): 0f55a9eb73b > author: Aleksey Midenkov <midenok(a)gmail.com> > committer: Aleksey Midenkov <midenok(a)gmail.com> > timestamp: 2019-06-27 18:05:25 +0300 > message: > > MDEV-16222 Assertion `0' failed in row_purge_remove_sec_if_poss_leaf on table with virtual columns and indexes > > Cause > Stale thd->m_stmt_da->m_sql_errno which is from different invocation. > > Fix > Reset error state before attempt to open table. > > diff --git a/mysql-test/suite/innodb/r/purge_secondary_mdev-16222.result b/mysql-test/suite/innodb/r/purge_secondary_mdev-16222.result > new file mode 100644 > index 00000000000..30e8f9800fb > --- /dev/null > +++ b/mysql-test/suite/innodb/r/purge_secondary_mdev-16222.result > @@ -0,0 +1,34 @@ ... > +select * from t1 into outfile 'load.data'; > +Warnings: > +Warning 1287 '<select expression> INTO <destination>;' is deprecated and will be removed in a future release. Please use 'SELECT <select list> INTO <destination> FROM...' instead See the warning, better not to use the deprecated syntax, unless you specifically want to test it, > +load data infile 'load.data' replace into table t1; > +set debug_sync= "now WAIT_FOR latch_released"; > +set global debug_dbug= "-d,ib_purge_virtual_mdev_16222_1"; > diff --git a/sql/sql_class.cc b/sql/sql_class.cc > index d65b6b8ed8d..cb73b7db2de 100644 > --- a/sql/sql_class.cc > +++ b/sql/sql_class.cc > @@ -4762,7 +4762,10 @@ TABLE *open_purge_table(THD *thd, const char *db, size_t dblen, > DBUG_ASSERT(!error || !ot_ctx.can_recover_from_failed_open()); > > if (unlikely(error)) > + { > close_thread_tables(thd); > + thd->clear_error(); We use thd->clear_error() in many places, but over years there were quite a few problems with it, and now I'd prefer to avoid it whenever possible. The thing is, you want to remove a specific error on a specific code path, but thd->clear_error() removes all errors, and no matter how you got here. Generally intercepting errors with Internal_error_handler is a much safer approach. In this case, though, thd->clear_error() might be okay, but better move it up the stack, do it in the caller when there can be only one way to reach this code. Not in a kind-of-generically-looking open table helper. > + } > > DBUG_RETURN(error ? NULL : tl->table); > } > diff --git a/sql/table.cc b/sql/table.cc > index 9ac167b6adb..93c2c5e88a8 100644 > --- a/sql/table.cc > +++ b/sql/table.cc > @@ -8240,6 +8240,7 @@ int TABLE::update_virtual_fields(handler *h, enum_vcol_update_mode update_mode) > > int TABLE::update_virtual_field(Field *vf) > { > + DBUG_ASSERT(!in_use->is_error()); Yes, that's what I mean above. There were cases when update_virtual_field() was called when is_error() was true and is_error() introduced a bug. I don't quite remember details, but it was something like, after the error has happened somewhere, on the way to returning an error, it was calling update_virtual_field() and some code in a function down the stack was, like some_function_which_returns_void(....); if (thd->is_error()) // the error inside a function { ... handle it ... } and the error in this case was not caused by anything in the function, but existed from before. The point is, is_error() and clear_error() are global, and you're generally interested in local state. Whether _this block_ caused an error. Clear the error caused by _this function call_. And so on, not clear an error that might've happened some time somewhere at the undefined point in the past before this code line was reached. > Query_arena backup_arena; > DBUG_ENTER("TABLE::update_virtual_field"); > in_use->set_n_backup_active_arena(expr_arena, &backup_arena); Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 2

Re: [Maria-developers] 23b70146c16: MDEV-6268 SPIDER table with no COMMENT clause causes queries to wait forever
by Sergei Golubchik 04 Jul '19

04 Jul '19

Hi, Kentoku! On Jul 04, Kentoku wrote: > revision-id: 23b70146c16 (mariadb-10.4.5-21-g23b70146c16) > parent(s): 24773bf3802 > author: Kentoku <kentokushiba(a)gmail.com> > committer: Kentoku <kentokushiba(a)gmail.com> > timestamp: 2019-06-10 13:59:18 +0900 > message: > > MDEV-6268 SPIDER table with no COMMENT clause causes queries to wait forever > Add mysql_mutex_timedlock() and add the following parameter to Spider > - spider_internal_lock_wait_timeout > The timeout when Spider tries to get internal locks. > 0 or more : the tomeout. (second) > The default value is -1 please, don't forget to leave an empty line after the first line in a commit comment. That's how various git tools (github included) separate "commit subject" from the "commit comment body". Git itself doesn't care, but it's a convention that some git-related tools rely on. Now, about the fix. Svoj made a general obervation on Slack that it's much better to eliminate or detect deadlocks, instead of wait them out with timeouts. And here we were able to find a rather cheap solution that would avoid such deadlocks in Spider. Here, how it can work: * On startup Spider calculates the unique spider instance ID by combining my_gethwaddr() and getpid(). With delimiters it might look something like "-d46d6d7345c9:23050-". This is done only once on startup. * When Spider needs to open a table, it checks whether this THD has a user variable named `spider_parents` and it it does, whether this variable contains the above mentioned unique spider instance ID. This is just one strstr(), so fast, no complex parsing. If the string is found, Spider returns an error, doesn't open a table. * When Spider connects to another host now it sends show table status from `test` like 'user' in the new approach it'll send set @spider_parents="${its current spider_parents}-d46d6d7345c9:23050-"; show table status from `test` like 'user' this is all in one packet, so no additional round-trips. Here `${its current spider_parents}` is the value of @spider_parents on the connecting host. This is it, it'll prevent self-connects, both direct and indirect via a chain of spider tables. Because @spider_parents variable is user-visible, better to use a different name for it, something that has a smaller chance of causing a conflict. May be @insternal_spider_deadlock_prevention_parent_list :) Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 1

[Maria-developers] Weekly Report
by Rucha Deodhar 01 Jul '19

01 Jul '19

Report for week 5: Hello! This week I was able to make INSERT...SELECT...RETURNING return the listed fields from the INSERT table. From the parsing stage in sql_parse.cc, the context and the first table gets masked , so I stored the previous values in another variable just before they are masked and swapped it before and after calling setup_fields() and setup_wild in select_insert :: prepare(). This makes the items and expressions in returning_list point to insert table before the above functions are called, and then swapping it back for the rest of the prepare() function. Here is the link to test cases and code: Test and result: https://github.com/MariaDB/server/commit/0792ba33a1b50e7beab3b7e05d36df29e7… Code: https://github.com/MariaDB/server/commit/4d1d37f2bdca398ff3ace804cfb3b5a940… I also tested my code for some complex queries like subquery in RETURNING and checked if it outputs error message when invalid expressions are used. It was working correctly. Here is the link for the same: Error message: https://github.com/MariaDB/server/commit/9ffe813eb82dc0125def187e07dbd5168b… Subqueries: https://github.com/MariaDB/server/commit/2363cb3993d86685ff1402cfa7ee050dd7… I was also thinking about other ways to get the items and expressions in returning_list, this is one of the ways in which it could be implemented. Here is the link to GDoc: https://docs.google.com/document/d/1TZdlL_V9MEbPTuXNwpzR0t9ciNeV25AeUIvSRXH… I understand this is too big a change for the grammar and I am trying to find alternate solution. I also changed the variable names and made them more descriptive. Regards, Rucha Deodhar

1 0

Re: [Maria-developers] [Commits] 61ad3ef: MDEV-19820 Wrong result with multiple single column index request
by Marko Mäkelä 25 Jun '19

25 Jun '19

Hi Igor, On Sun, Jun 23, 2019 at 8:26 PM IgorBabaev <igor(a)mariadb.com> wrote: > > revision-id: 61ad3efea00303e2d0cd85628c76251b08e79531 (mariadb-10.4.4-194-g61ad3ef) > parent(s): 8b576616b442d061356bc5a2abd410f478e98ee7 > author: Igor Babaev > committer: Igor Babaev > timestamp: 2019-06-23 10:26:42 -0700 > message: > > MDEV-19820 Wrong result with multiple single column index request > > The bug occured when the optimizer decided to use a rowid filter built > by a range index scan to access an InnoDB table with generated clustered > index. > When a table is accessed by a secondary index Idx employing a rowid filter the > the value of pk contained in the found index tuple is checked against the > filter. A call of the handler function position is supposed to put the > pk value into the handler::ref buffer. However for generated clustered > primary keys it did not happened. The patch fixes this problem. [snip] > diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc > index 659a824..d796b88 100644 > --- a/storage/innobase/row/row0sel.cc > +++ b/storage/innobase/row/row0sel.cc > @@ -3902,10 +3902,23 @@ row_search_idx_cond_check( > > switch (result) { > case ICP_MATCH: > - if (handler_rowid_filter_is_active(prebuilt->pk_filter) > - && !handler_rowid_filter_check(prebuilt->pk_filter)) { > - MONITOR_INC(MONITOR_ICP_MATCH); > - return(ICP_NO_MATCH); > + if (handler_rowid_filter_is_active(prebuilt->pk_filter)) { > + if (prebuilt->clust_index_was_generated) { > + ulint len; > + dict_index_t* index = prebuilt->index; > + const byte* data = rec_get_nth_field( > + rec, offsets, index->n_fields - 1, > + &len); > + ut_ad(dict_index_get_nth_col(index, > + index->n_fields - 1) > + ->prtype == (DATA_ROW_ID | DATA_NOT_NULL)); > + ut_ad(len == DATA_ROW_ID_LEN); > + memcpy(prebuilt->row_id, data, DATA_ROW_ID_LEN); > + } > + if (!handler_rowid_filter_check(prebuilt->pk_filter)) { > + MONITOR_INC(MONITOR_ICP_MATCH); > + return(ICP_NO_MATCH); > + } > } > /* Convert the remaining fields to MySQL format. > If this is a secondary index record, we must defer Here, the changed code is assuming that prebuilt->index is a secondary index without explicitly saying so. I do not see an assertion similar to ut_ad(!handler_rowid_filter_is_active(prebuilt->pk_filter) || !prebuilt->index->is_primary()); anywhere in row0sel.cc or ha_innodb.cc. A natural place for such a check could be in ha_innobase::build_template(). If the pk_filter is only being used for secondary index access, then I think that it should be enforced by some debug assertion and with a test case. Another problem is that we are doing busy work if handler_rowid_filter_check(prebuilt->pk_filter) does hold, and we will not return ICP_NO_MATCH above. I wonder if a simpler approach of reordering and deduplicating some code would work: diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 659a824a0d5..068dc849449 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -3902,11 +3902,6 @@ row_search_idx_cond_check( switch (result) { case ICP_MATCH: - if (handler_rowid_filter_is_active(prebuilt->pk_filter) - && !handler_rowid_filter_check(prebuilt->pk_filter)) { - MONITOR_INC(MONITOR_ICP_MATCH); - return(ICP_NO_MATCH); - } /* Convert the remaining fields to MySQL format. If this is a secondary index record, we must defer this until we have fetched the clustered index record. */ @@ -3920,6 +3915,10 @@ row_search_idx_cond_check( } } MONITOR_INC(MONITOR_ICP_MATCH); + if (handler_rowid_filter_is_active(prebuilt->pk_filter) + && !handler_rowid_filter_check(prebuilt->pk_filter)) { + result = ICP_NO_MATCH; + } return(result); case ICP_NO_MATCH: MONITOR_INC(MONITOR_ICP_NO_MATCH); Can you try that? With best regards, Marko

1 0

[Maria-developers] Weekly Report
by Rucha Deodhar 24 Jun '19

24 Jun '19

Report for week 4: Hello! This week I was able to make INSERT...RETURNING work. It is working fine for simple insert statement, inserting multiple rows in single statement, INSERT...SET and INSERT...ON DUPLICATE KEY UPDATE. It is returning the listed fields, working for alias and operators as well. I was getting segmentation fault when I was working on this. Zulip messages helped to solve that problem by cleaning returning_list by calling returning_list.empty() in mysql_init_select(). I also realised I hadn't called setup_fields(), setup_wild() and checking if there is '*' for returning_list in mysql_prepare_insert() which was also causing this error. Here is a link to the test cases and results for above variants of INSERT...RETURNING: https://github.com/rucha174/server/commit/66f8046d615f97a1078c8eedc3cf11d6e… As for INSERT...SELECT...RETURNING, I passed lex->field_list instead of returning_list as an argument in prepare(), send_result_set_metadata() and send_data(). It was returning all the fields from the insert table. Then I passed returning_list as the argument and it is returning the listed fields but from the select table. So now I am working on it by looking how select_insert class uses lex->field_list and do the same for returning_list. Here is the link to code which returns the listed fields for select table: https://github.com/rucha174/server/commit/6b18dd3cfc76d41871357c5b2006e04d1… Regards, Rucha Deodhar

1 0

[Maria-developers] List of the most used plugins
by GUESNET, ETIENNE (ext) 21 Jun '19

21 Jun '19

Hi, I am porting MariaDB on AIX, and some plugins do not work. As I am not a MariaDB user, I do not know which one is the most important or the most used. To prioritize correctly the port, do you have some statistic, or just a list of key plugins? I have ported for now archive, federatedX, blackhole, partition, InnoDB, aria/myisam, feedback, locales and some auth_* plugins. Thanks, Etienne.

1 0

[Maria-developers] review for MDEV-14564: Support FOR loop in stored aggregate functions
by Alexander Barkov 20 Jun '19

20 Jun '19

Hi Varun, I was recently assigned as a reviewer for this patch: http://lists.askmonty.org/pipermail/commits/2017-December/011777.html I have some suggestions. 1. The MDEV says the proposed syntax is: FOR GROUP ROWS DO and you implemented: FOR GROUP NEXT ROW DO I think the syntax on MDEV looks better. 2. I don't like that you're adding a new uint member sp_instr_agg_cfetch::m_dest An instruction processing a 'FETCH GROUP NEXT ROW' statement does not need it. 3. The generated code: +show function code f1; +Pos Instruction +0 set total at 1 0 +1 jump_if_not 5(5) 1 +2 +3 set total@1 total@1 + a@0 +4 jump 1 +5 freturn int total at 1 does not look nice. - The command at position 2 must have a name. - The command at position 1 looks like a hack. The underlying code is: + if (! it->val_bool() || thd->spcont->forced_error) So it is a conditional jump which never jumps on the condition, i.e. val_bool(). It jumps only on thd->spcont->forced_error. Please create a new class sp_instr_agg_cfetch_or_jump, derive it from sp_instr, and add m_dest to this class. Please implement print() and execute() for it. print() should display: - the name of the command, "agg_cfetch_or_jump". - the label where it jumps to when 'no rows' happens. Note, sp_instr_agg_cfetch_or_jump will have almost nothing to share with the old sp_instr_agg_cfetch. So sp_instr_agg_cfetch_or_jump should derive from sp_instr, not from sp_instr_agg_cfetch. Please perform jump to a proper position on 'no more rows' just inside sp_instr_agg_cfetch_or_jump::execute() So the code should look about like this: Pos Instruction 0 set total at 1 0 1 agg_cfetch_or_jump 4 2 set total@1 total@1 + a@0 3 jump 1 4 freturn int total at 1 4. As discussed on slack, the name for the command sp_instr_agg_cfetch was probably a not good choise. The "c" in other commands like: - sp_instr_copen - sp_instr_cfetch - sp_instr_cclose is an abbreviation for the word "cursor". sp_instr_agg_cfetch does not use any cursors (i.e. sp_cursor instances). Please rename sp_instr_agg_cfetch to sp_instr_fetch_group_row. And the new command can be named something like: sp_instr_fetch_group_row_or_jump, instead of sp_instr_agg_cfetch_or_jump. Greetings.

1 1

Re: [Maria-developers] [Commits] 8b545d3d414: MDEV-7409 On RBR, extend the PROCESSLIST info to include at least the name of the recently used table
by andrei.elkin＠pp.inet.fi 19 Jun '19

19 Jun '19

Sachin, salve. > revision-id: 8b545d3d41416772203cb98f06243873a4f0d8f7 (mariadb-10.4.5-24-g8b545d3d414) > parent(s): e35676f5557d68c7b51ba47aa73dcdf72eafa436 > author: Sachin > committer: Sachin > timestamp: 2019-05-31 12:02:37 +0530 > message: > > MDEV-7409 On RBR, extend the PROCESSLIST info to include at least the name of the recently used table > > When RBR is used, add the db name to db Field and table name to Status > Field of the "SHOW FULL PROCESSLIST" command for SQL thread. So the patch deals with the State column of the Show-Processlist command. I would endorse an output like the following: MariaDB [test]> show processlist; +----+-------------+-----------------+------+-----------+------+--------------------------------------------------+-----------------------------------------------+----------+ | Id | User | Host | db | Command | Time | State | Info | Progress | +----+-------------+-----------------+------+-----------+------+--------------------------------------------------+-----------------------------------------------+----------+ ... | 13 | system user | | test | Slave_SQL | 101 | Write_rows_log_event::write_row(-1) on table `t` | GTID 0-1-17: insert into `t` values (46),(47) | 0.000 | +----+-------------+-----------------+------+-----------+------+--------------------------------------------------+-----------------------------------------------+----------+ notice the ANSI quotes which could be more than for pretty printing considering legal names like `[:space:]+t`. Alas the text from Annotate event does not quote the table name so the actual Info is not that perfect *also* because the actual one does not display GTID.. Could we make that - the Annotate event quoted names and GTID into Progress-list Info in a separate patch for 10.5? And also to turn Progress column meaningful. It's always zero, but could print for the user the executed fraction of the whole event (sure we need to know in the Rows_log_event metadata how much rows the statement changed on the master; on slave we would just count the executed ones). > > --- > mysql-test/suite/rpl/r/rpl_rbr_monitor.result | 57 +++++++++++++++++ > mysql-test/suite/rpl/t/rpl_rbr_monitor.test | 86 +++++++++++++++++++++++++ > sql/debug_sync.cc | 3 +- > sql/log_event.cc | 90 +++++++++++++++++++++------ > 4 files changed, 215 insertions(+), 21 deletions(-) > To the test, you could really do it simpler without > --- /dev/null > +++ b/mysql-test/suite/rpl/t/rpl_rbr_monitor.test > @@ -0,0 +1,86 @@ > +--source include/have_innodb.inc > +--source include/have_debug.inc any debug. Just start a slave local trx to block Rows_log_event applying at some (see $param) point: --connection slave begin; --eval select a from t where a > $param for update; > +--source include/have_binlog_format_row.inc > +--source include/master-slave.inc > +--enable_connect_log > + > +--connection master > +create table t1(a int primary key); > +--save_master_pos > + > +--connection slave > +--sync_with_master > +SET GLOBAL debug_dbug="+d,should_wait_for_mdev7409"; > +select * from t1; > + > +--connection master > +insert into t1(a) values(1); > +--save_master_pos > + > +--echo #monitoring write rows > +--connection slave > + > +let $wait_condition= SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Write_rows_log_event::write_row(%) on table t1"; > +--source include/wait_condition.inc > +SELECT db , state FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Write_rows_log_event::write_row(%) on table t1"; > +set debug_sync="now signal cont"; > +--sync_with_master > + > +--echo #monitoring update rows > +--connection master > +update t1 set a = a + 4194304 ; > + > +--connection slave > +let $wait_condition= SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Update_rows_log_event::find_row(%) on table t1"; > +--source include/wait_condition.inc > +SELECT db, state FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Update_rows_log_event::find_row(%) on table t1"; > +set debug_sync="now signal cont1"; > + > +let $wait_condition= SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Update_rows_log_event::unpack_current_row(%) on table t1"; > +--source include/wait_condition.inc > +SELECT db, state FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Update_rows_log_event::unpack_current_row(%) on table t1"; > +set debug_sync="now signal cont2"; > + > +let $wait_condition= SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Update_rows_log_event::ha_update_row(%) on table t1"; > +--source include/wait_condition.inc > +SELECT db, state FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Update_rows_log_event::ha_update_row(%) on table t1"; > +set debug_sync="now signal cont3"; > +set debug_sync="RESET"; > +--sync_with_master > + > +--echo #monitoring delete rows > +--connection master > +delete from t1 where a>1; > + > +--connection slave > +let $wait_condition= SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Delete_rows_log_event::find_row(%) on table t1"; > +--source include/wait_condition.inc > +SELECT db , state FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Delete_rows_log_event::find_row(%) on table t1"; > +set debug_sync="now signal cont1"; > + > +let $wait_condition= SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Delete_rows_log_event::ha_delete_row(%) on table t1"; > +--source include/wait_condition.inc > +SELECT db, state FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Delete_rows_log_event::ha_delete_row(%) on table t1"; > +set debug_sync="now signal cont2"; > +set debug_sync="RESET"; > +--sync_with_master > +SET GLOBAL debug_dbug=""; > + > +#CleanUp > +--connection master > +drop table t1; > +--sync_slave_with_master > + > +--source include/rpl_end.inc > diff --git a/sql/debug_sync.cc b/sql/debug_sync.cc That would make all the simulation and synchronization in the source code incl the following changes unnecessary: > index bf721bddb85..7ca8d8a8ecb 100644 > --- a/sql/debug_sync.cc > +++ b/sql/debug_sync.cc > @@ -1360,7 +1360,8 @@ static void debug_sync_execute(THD *thd, st_debug_sync_action *action) > Do this before emitting the signal, so other threads can see it > if they awake before we enter_cond() below. > */ > - if (action->wait_for.length()) > + if (action->wait_for.length() && > + DBUG_EVALUATE_IF("should_wait_for_mdev7409", 0, 1)) > { > st_debug_sync_control *ds_control= thd->debug_sync_control; > strxnmov(ds_control->ds_proc_info, sizeof(ds_control->ds_proc_info)-1, > diff --git a/sql/log_event.cc b/sql/log_event.cc > index f25ebd56792..8c944a6e282 100644 > --- a/sql/log_event.cc > +++ b/sql/log_event.cc > @@ -54,6 +54,7 @@ > #include "rpl_constants.h" > #include "sql_digest.h" > #include "zlib.h" > +#include "debug_sync.h" > > #define my_b_write_string(A, B) my_b_write((A), (uchar*)(B), (uint) (sizeof(B) - 1)) > > @@ -13693,19 +13694,29 @@ Write_rows_log_event::do_exec_row(rpl_group_info *rgi) > { > DBUG_ASSERT(m_table != NULL); > const char *tmp= thd->get_proc_info(); > - const char *message= "Write_rows_log_event::write_row()"; > + LEX_CSTRING *tmp_db= &thd->db; > + char *message, msg[128]; > + my_snprintf(msg, sizeof(msg),"Write_rows_log_event::write_row() on table %s", > + m_table->s->table_name.str); > + thd->set_db(&m_table->s->db); > + message= msg; > int error; > > #ifdef WSREP_PROC_INFO > my_snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1, > - "Write_rows_log_event::write_row(%lld)", > - (long long) wsrep_thd_trx_seqno(thd)); > + "Write_rows_log_event::write_row(%lld) on table %s", > + (long long) wsrep_thd_trx_seqno(thd), m_table->s->table_name.str); > message= thd->wsrep_info; > #endif /* WSREP_PROC_INFO */ > > thd_proc_info(thd, message); > + DBUG_EXECUTE_IF("should_wait_for_mdev7409",{ > + debug_sync_set_action > + (thd, STRING_WITH_LEN("now WAIT_FOR cont")); > + };); > error= write_row(rgi, slave_exec_mode == SLAVE_EXEC_MODE_IDEMPOTENT); > thd_proc_info(thd, tmp); > + thd->set_db(tmp_db); > > if (unlikely(error) && unlikely(!thd->is_error())) > { > @@ -14368,32 +14379,47 @@ int Delete_rows_log_event::do_exec_row(rpl_group_info *rgi) > { > int error; > const char *tmp= thd->get_proc_info(); > - const char *message= "Delete_rows_log_event::find_row()"; > + LEX_CSTRING *tmp_db= &thd->db; > + char *message, msg[128]; > + my_snprintf(msg, sizeof(msg),"Delete_rows_log_event::find_row() on table %s", > + m_table->s->table_name.str); > + thd->set_db(&m_table->s->db); > + message= msg; > const bool invoke_triggers= > slave_run_triggers_for_rbr && !master_had_triggers && m_table->triggers; > DBUG_ASSERT(m_table != NULL); > > #ifdef WSREP_PROC_INFO > my_snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1, > - "Delete_rows_log_event::find_row(%lld)", > - (long long) wsrep_thd_trx_seqno(thd)); > + "Delete_rows_log_event::find_row(%lld) on table %s", > + (long long) wsrep_thd_trx_seqno(thd), m_table->s->table_name.str) ; > message= thd->wsrep_info; > #endif /* WSREP_PROC_INFO */ > > thd_proc_info(thd, message); > + DBUG_EXECUTE_IF("should_wait_for_mdev7409",{ > + debug_sync_set_action > + (thd, STRING_WITH_LEN("now WAIT_FOR cont1")); > + };); > if (likely(!(error= find_row(rgi)))) > { > /* > Delete the record found, located in record[0] > */ > - message= "Delete_rows_log_event::ha_delete_row()"; > + my_snprintf(msg, sizeof(msg),"Delete_rows_log_event::ha_delete_row() on table %s", > + m_table->s->table_name.str); > + message= msg; > #ifdef WSREP_PROC_INFO > snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1, > - "Delete_rows_log_event::ha_delete_row(%lld)", > - (long long) wsrep_thd_trx_seqno(thd)); > + "Delete_rows_log_event::ha_delete_row(%lld) on table %s", > + (long long) wsrep_thd_trx_seqno(thd), m_table->s->table_name.str) ; > message= thd->wsrep_info; > #endif > thd_proc_info(thd, message); > + DBUG_EXECUTE_IF("should_wait_for_mdev7409",{ > + debug_sync_set_action > + (thd, STRING_WITH_LEN("now WAIT_FOR cont2")); > + };); > > if (invoke_triggers && > unlikely(process_triggers(TRG_EVENT_DELETE, TRG_ACTION_BEFORE, FALSE))) > @@ -14422,6 +14448,7 @@ int Delete_rows_log_event::do_exec_row(rpl_group_info *rgi) > m_table->file->ha_index_or_rnd_end(); > } > thd_proc_info(thd, tmp); > + thd->set_db(tmp_db); > return error; > } > > @@ -14590,17 +14617,27 @@ Update_rows_log_event::do_exec_row(rpl_group_info *rgi) > const bool invoke_triggers= > slave_run_triggers_for_rbr && !master_had_triggers && m_table->triggers; > const char *tmp= thd->get_proc_info(); > - const char *message= "Update_rows_log_event::find_row()"; > + LEX_CSTRING *tmp_db= &thd->db; > + char *message, msg[128]; > DBUG_ASSERT(m_table != NULL); > + my_snprintf(msg, sizeof(msg),"Update_rows_log_event::find_row() on table %s", > + m_table->s->table_name.str); > + thd->set_db(&m_table->s->db); > + message= msg; > > #ifdef WSREP_PROC_INFO > my_snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1, > - "Update_rows_log_event::find_row(%lld)", > - (long long) wsrep_thd_trx_seqno(thd)); > + "Update_rows_log_event::find_row(%lld) on table %s", > + (long long) wsrep_thd_trx_seqno(thd), m_table->s->table_name.str) ; > message= thd->wsrep_info; > #endif /* WSREP_PROC_INFO */ > > thd_proc_info(thd, message); > + DBUG_EXECUTE_IF("should_wait_for_mdev7409",{ > + debug_sync_set_action > + (thd, STRING_WITH_LEN("now WAIT_FOR cont1")); > + };); > + > int error= find_row(rgi); > if (unlikely(error)) > { > @@ -14611,6 +14648,7 @@ Update_rows_log_event::do_exec_row(rpl_group_info *rgi) > if ((m_curr_row= m_curr_row_end)) > unpack_current_row(rgi, &m_cols_ai); > thd_proc_info(thd, tmp); > + thd->db= *tmp_db; > return error; > } > > @@ -14628,16 +14666,22 @@ Update_rows_log_event::do_exec_row(rpl_group_info *rgi) > store_record(m_table,record[1]); > > m_curr_row= m_curr_row_end; > - message= "Update_rows_log_event::unpack_current_row()"; > + my_snprintf(msg, sizeof(msg),"Update_rows_log_event::unpack_current_row() on table %s", > + m_table->s->table_name.str); > + message= msg; > #ifdef WSREP_PROC_INFO > my_snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1, > - "Update_rows_log_event::unpack_current_row(%lld)", > - (long long) wsrep_thd_trx_seqno(thd)); > + "Update_rows_log_event::unpack_current_row(%lld) on table %s", > + (long long) wsrep_thd_trx_seqno(thd), m_table->s->table_name.str) ; > message= thd->wsrep_info; > #endif /* WSREP_PROC_INFO */ > > /* this also updates m_curr_row_end */ > thd_proc_info(thd, message); > + DBUG_EXECUTE_IF("should_wait_for_mdev7409",{ > + debug_sync_set_action > + (thd, STRING_WITH_LEN("now WAIT_FOR cont2")); > + };); > if (unlikely((error= unpack_current_row(rgi, &m_cols_ai)))) > goto err; > > @@ -14655,15 +14699,21 @@ Update_rows_log_event::do_exec_row(rpl_group_info *rgi) > DBUG_DUMP("new values", m_table->record[0], m_table->s->reclength); > #endif > > - message= "Update_rows_log_event::ha_update_row()"; > + my_snprintf(msg, sizeof(msg),"Update_rows_log_event::ha_update_row() on table %s", > + m_table->s->table_name.str); > + message= msg; > #ifdef WSREP_PROC_INFO > my_snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1, > - "Update_rows_log_event::ha_update_row(%lld)", > - (long long) wsrep_thd_trx_seqno(thd)); > + "Update_rows_log_event::ha_update_row(%lld) on table %s", > + (long long) wsrep_thd_trx_seqno(thd), m_table->s->table_name.str) ; > message= thd->wsrep_info; > #endif /* WSREP_PROC_INFO */ > > thd_proc_info(thd, message); > + DBUG_EXECUTE_IF("should_wait_for_mdev7409",{ > + debug_sync_set_action > + (thd, STRING_WITH_LEN("now WAIT_FOR cont3")); > + };); > if (invoke_triggers && > unlikely(process_triggers(TRG_EVENT_UPDATE, TRG_ACTION_BEFORE, TRUE))) > { > @@ -14693,9 +14743,9 @@ Update_rows_log_event::do_exec_row(rpl_group_info *rgi) > unlikely(process_triggers(TRG_EVENT_UPDATE, TRG_ACTION_AFTER, TRUE))) > error= HA_ERR_GENERIC; // in case if error is not set yet > > - thd_proc_info(thd, tmp); > - > err: > + thd_proc_info(thd, tmp); > + thd->set_db(tmp_db); > m_table->file->ha_index_or_rnd_end(); > return error; > } Cheers, Andrei

2 1

[Maria-developers] Weekly Report
by Rucha Deodhar 17 Jun '19

17 Jun '19

Report for week 3: Hello! This week was more productive than the last one. Zulip messages helped to extend parser for INSERT...SELECT...RETURNING. I created SELECT_LEX :: returning_list and swapped it with SELECT_LEX :: item_list. I have also added test cases and result for the same. Feedback of GDoc and zulip messages helped in understanding the code better. I have started implementing INSERT...RETURNING. I am able to check whether there is result set in the query. But during result->send_result_set_metadata() server crashes and I get segmentation fault. I am trying to fix it. Here is the link to my last commit of code and test cases code: https://github.com/MariaDB/server/commit/d06f21b84cc65424a795c21cc3fee5546e… Test case: https://github.com/MariaDB/server/commit/280a885f1563594678141a360305a253b4… Result: https://github.com/MariaDB/server/commit/218d81b86d941a874cfac9123d7aeb141e… Regards, Rucha Deodhar

1 0

Re: [Maria-developers] b4cdf20dd15: MDEV-17591 Create MariaDB named commands/symlinks
by Sergei Golubchik 14 Jun '19

14 Jun '19

Hi, Rasmus! On Jun 14, Rasmus Johansson wrote: > revision-id: b4cdf20dd15 (mariadb-10.4.5-44-gb4cdf20dd15) > parent(s): 8e3a4be45c5 > author: Rasmus Johansson <rasmus(a)mariadb.com> > committer: Rasmus Johansson <rasmus(a)mariadb.com> > timestamp: 2019-06-14 14:55:20 +0000 > message: > > MDEV-17591 Create MariaDB named commands/symlinks > > diff --git a/cmake/symlinks.cmake b/cmake/symlinks.cmake > new file mode 100644 > index 00000000000..32cf24e3e3b > --- /dev/null > +++ b/cmake/symlinks.cmake > @@ -0,0 +1,76 @@ > +# MariaDB names for executables > +list(APPEND MARIADB_SYMLINK_NAMES "mysql" "mariadb") > +list(APPEND MARIADB_SYMLINK_NAMES "mysqlaccess" "mariadb-access") > +list(APPEND MARIADB_SYMLINK_NAMES "mysqladmin" "mariadb-admin") > +list(APPEND MARIADB_SYMLINK_NAMES "mariabackup" "mariadb-backup") ... > +# Add MariaDB symlinks > +macro(CREATE_MARIADB_SYMLINK src comp) > + # Find the MariaDB name for executable > + list(FIND MARIADB_SYMLINK_NAMES ${src} _index) > + > + if (${_index} GREATER -1) > + MATH(EXPR _index "${_index}+1") > + list(GET MARIADB_SYMLINK_NAMES ${_index} _name) > + set(mariadbname ${_name}) > + endif() > + > + if (mariadbname) > + set(dest ${mariadbname}) > + set(symlink_install_dir ${INSTALL_BINDIR}) > + > + # adjust install location if needed > + if(${dest} MATCHES "mariadb-install-db") > + set(symlink_install_dir ${INSTALL_SCRIPTDIR}) > + endif() I still think it's very fragile and difficult to maintain. Why not to pass the destination as a third argument to CREATE_MARIADB_SYMLINK? As far as I can see, everywhere where you use CREATE_MARIADB_SYMLINK the destination is available. The rest is okay. I'd still suggest to simplify by splitting the list in two, like macro(REGISTER_SYMLINK from to) list(APPEND MARIADB_SYMLINK_FROMS from) list(APPEND MARIADB_SYMLINK_TOS to) endmacro() REGISTER_SYMLINK(mysql mariadb) REGISTER_SYMLINK(mysqladmin mariadb-access) ...etc... Then you won't need to do math on indexes, you simply search in MARIADB_SYMLINK_FROMS and get the same element in MARIADB_SYMLINK_TOS. Also you will need to search only in MARIADB_SYMLINK_FROMS, not like now, when you search in the complete list, risking to get spurious matches on mariadb* names. Same in manpages, you'll only iterate over MARIADB_SYMLINK_FROMS list. > + > + CREATE_MARIADB_SYMLINK_IN_DIR(${src} ${dest} ${symlink_install_dir} ${comp}) > + endif() > +endmacro(CREATE_MARIADB_SYMLINK) Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

1 0

Re: [Maria-developers] e35fb718eae: MDEV-17591 Create MariaDB named commands/symlinks
by Sergei Golubchik 14 Jun '19

14 Jun '19

Hi, Rasmus! On Jun 13, Rasmus Johansson wrote: > revision-id: e35fb718eae (mariadb-10.4.5-44-ge35fb718eae) > parent(s): 8e3a4be45c5 > author: Rasmus Johansson <razze(a)iki.fi> > committer: Rasmus Johansson <razze(a)iki.fi> Is that an address you want to have in git history? You already have two, just making sure the third one is intentional. > timestamp: 2019-06-13 13:20:40 +0000 > message: > > MDEV-17591 Create MariaDB named commands/symlinks > > diff --git a/cmake/install_macros.cmake b/cmake/install_macros.cmake > index f28720e97bc..e49f296ed75 100644 > --- a/cmake/install_macros.cmake > +++ b/cmake/install_macros.cmake > @@ -111,8 +111,10 @@ FUNCTION(INSTALL_SCRIPT) > ENDIF() > > INSTALL(PROGRAMS ${script} DESTINATION ${ARG_DESTINATION} ${COMP}) > + string(REPLACE "${CMAKE_BINARY_DIR}/scripts/" "" dest ${script}) better not to assume that the script is in ${CMAKE_BINARY_DIR}/scripts/ I'd use get_filename_component() here. Like get_filename_component(dest "${scripts}" NAME) > + CREATE_MARIADB_SYMLINK(${dest}) > > - INSTALL_MANPAGE(${script}) > + INSTALL_MANPAGE(${dest}) > ENDFUNCTION() > > > diff --git a/cmake/mysql_add_executable.cmake b/cmake/mysql_add_executable.cmake > index d3a888f9a75..ad7320cd193 100644 > --- a/cmake/mysql_add_executable.cmake > +++ b/cmake/mysql_add_executable.cmake > @@ -63,6 +63,9 @@ FUNCTION (MYSQL_ADD_EXECUTABLE) > UNSET(EXCLUDE_FROM_ALL) > ENDIF() > ADD_EXECUTABLE(${target} ${WIN32} ${MACOSX_BUNDLE} ${EXCLUDE_FROM_ALL} ${sources}) > + # create mariadb named symlink > + CREATE_MARIADB_SYMLINK(${target}) > + > # tell CPack where to install > IF(NOT ARG_EXCLUDE_FROM_ALL) > IF(NOT ARG_DESTINATION) > diff --git a/cmake/symlinks.cmake b/cmake/symlinks.cmake > new file mode 100644 > index 00000000000..0d0c637df5a > --- /dev/null > +++ b/cmake/symlinks.cmake > @@ -0,0 +1,74 @@ > +# MariaDB names for executables > +list(APPEND MARIADB_SYMLINK_NAMES "mysql" "mariadb") > +list(APPEND MARIADB_SYMLINK_NAMES "mysqlaccess" "mariadb-access") > +list(APPEND MARIADB_SYMLINK_NAMES "mysqladmin" "mariadb-admin") > +list(APPEND MARIADB_SYMLINK_NAMES "mariabackup" "mariadb-backup") > +list(APPEND MARIADB_SYMLINK_NAMES "mysqlbinlog" "mariadb-binlog") > +list(APPEND MARIADB_SYMLINK_NAMES "mysqlcheck" "mariadb-check") > +list(APPEND MARIADB_SYMLINK_NAMES "mysql_client_test_embedded" "mariadb-client-test-embedded") > +list(APPEND MARIADB_SYMLINK_NAMES "mariadb_config" "mariadb-config") > +list(APPEND MARIADB_SYMLINK_NAMES "mysql_convert_table_format" "mariadb-convert-table-format") > +list(APPEND MARIADB_SYMLINK_NAMES "mysqldump" "mariadb-dump") > +list(APPEND MARIADB_SYMLINK_NAMES "mysqldumpslow" "mariadb-dumpslow") > +list(APPEND MARIADB_SYMLINK_NAMES "mysql_embedded" "mariadb-embedded") > +list(APPEND MARIADB_SYMLINK_NAMES "mysql_find_rows" "mariadb-find-rows") > +list(APPEND MARIADB_SYMLINK_NAMES "mysql_fix_extensions" "mariadb-fix-extensions") > +list(APPEND MARIADB_SYMLINK_NAMES "mysqlhotcopy" "mariadb-hotcopy") > +list(APPEND MARIADB_SYMLINK_NAMES "mysqlimport" "mariadb-import") > +list(APPEND MARIADB_SYMLINK_NAMES "mysql_install_db" "mariadb-install-db") > +list(APPEND MARIADB_SYMLINK_NAMES "mysql_ldb" "mariadb-ldb") > +list(APPEND MARIADB_SYMLINK_NAMES "mysql_plugin" "mariadb-plugin") > +list(APPEND MARIADB_SYMLINK_NAMES "mysql_secure_installation" "mariadb-secure-installation") > +list(APPEND MARIADB_SYMLINK_NAMES "mysql_setpermission" "mariadb-setpermission") > +list(APPEND MARIADB_SYMLINK_NAMES "mysqlshow" "mariadb-show") > +list(APPEND MARIADB_SYMLINK_NAMES "mysqlslap" "mariadb-slap") > +list(APPEND MARIADB_SYMLINK_NAMES "mysqltest" "mariadb-test") > +list(APPEND MARIADB_SYMLINK_NAMES "mysqltest_embedded" "mariadb-test-embedded") > +list(APPEND MARIADB_SYMLINK_NAMES "mytop" "mariadb-top") > +list(APPEND MARIADB_SYMLINK_NAMES "mysql_tzinfo_to_sql" "mariadb-tzinfo-to-sql") > +list(APPEND MARIADB_SYMLINK_NAMES "mysql_upgrade" "mariadb-upgrade") > +list(APPEND MARIADB_SYMLINK_NAMES "mysql_upgrade_service" "mariadb-upgrade-service") > +list(APPEND MARIADB_SYMLINK_NAMES "mysql_upgrade_wizard" "mariadb-upgrade-wizard") > +list(APPEND MARIADB_SYMLINK_NAMES "mysql_waitpid" "mariadb-waitpid") > +list(APPEND MARIADB_SYMLINK_NAMES "mysqld" "mariadbd") > +list(APPEND MARIADB_SYMLINK_NAMES "mysqld_multi" "mariadbd-multi") > +list(APPEND MARIADB_SYMLINK_NAMES "mysqld_safe" "mariadbd-safe") > +list(APPEND MARIADB_SYMLINK_NAMES "mysqld_safe_helper" "mariadbd-safe-helper") > + > +# Add MariaDB symlinks > +macro(CREATE_MARIADB_SYMLINK src) do you still skip Windows, as in the previous commit? I couldn't find it in this one. > + # Find the MariaDB name for executable > + list(FIND MARIADB_SYMLINK_NAMES ${src} _index) > + > + if (${_index} GREATER -1) > + MATH(EXPR _index "${_index}+1") > + list(GET MARIADB_SYMLINK_NAMES ${_index} _name) > + MESSAGE(${_name}) I suspect this was a debugging aid that you forgot to remove > + set(mariadbname ${_name}) > + endif() > + > + if (mariadbname) > + set(dest ${mariadbname}) > + message("src: ${src}") > + message("dest: ${dest}") > + message("CMAKE_CURRENT_BINARY_DIR: ${CMAKE_CURRENT_BINARY_DIR}") > + message("INSTALL_BINDIR: ${INSTALL_BINDIR}") and that too > + > + add_custom_target( > + symlink_${dest} ALL > + DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${dest} > + ) > + > + add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${dest} POST_BUILD > + COMMAND ${CMAKE_COMMAND} -E create_symlink ${src} ${dest} > + COMMENT "mklink ${src} -> ${dest}") > + > + set(symlink_install_dir ${INSTALL_BINDIR}) > + # adjust install location if needed > + if(${dest} MATCHES "mariadb-install-db") > + set(symlink_install_dir ${INSTALL_SCRIPTDIR}) > + endif() this is very fragile. better to take the path from the ${src} also with get_filename_component(). > + > + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${dest} DESTINATION ${symlink_install_dir} COMPONENT symlinks) > + endif() > +endmacro(CREATE_MARIADB_SYMLINK) > diff --git a/debian/mariadb-client-10.4.links b/debian/mariadb-client-10.4.links > index 4a504969246..b93d2d7f8fd 100644 > --- a/debian/mariadb-client-10.4.links > +++ b/debian/mariadb-client-10.4.links > @@ -1,6 +1,44 @@ > +usr/bin/mysql_find_rows usr/bin/mariadb-find-rows > +usr/bin/mysql_fix_extensions usr/bin/mariadb-fix-extenstions > +usr/bin/mysql_plugin usr/bin/mariadb-plugin > +usr/bin/mysql_waitpid usr/bin/mariadb-waitpid > +usr/bin/mysqlaccess usr/bin/mariadb-access > +usr/bin/mysqladmin usr/bin/mariadb-admin > +usr/bin/mysqlanalyze usr/bin/mariadb-analyze > +usr/bin/mysqlbinlog usr/bin/mariadb-binlog > +usr/bin/mysqlcheck usr/bin/mariadb-analyze > +usr/bin/mysqlcheck usr/bin/mariadb-optimize > +usr/bin/mysqlcheck usr/bin/mariadb-repair > usr/bin/mysqlcheck usr/bin/mysqlanalyze > usr/bin/mysqlcheck usr/bin/mysqloptimize > usr/bin/mysqlcheck usr/bin/mysqlrepair > +usr/bin/mysqldump usr/bin/mariadb-dump > +usr/bin/mysqldumpslow usr/bin/mariadb-dump-slow > +usr/bin/mysqlimport usr/bin/mariadb-import > +usr/bin/mysqloptimize usr/bin/mariadb-optimize > +usr/bin/mysqlrepair usr/bin/mariadb-repair > +usr/bin/mysqlreport usr/bin/mariadb-report > +usr/bin/mysqlslap usr/bin/mariadb-slap > +usr/bin/mysqltest usr/bin/mariadb-test > +usr/share/man/man1/mysql_find_rows.1.gz usr/share/man/man1/mariadb-find-rows.1.gz > +usr/share/man/man1/mysql_fix_extensions.1.gz usr/share/man/man1/mariadb-fix-extensions.1.gz manpages aren't symlinked in rpms or bintars. and symlinks depends only on MariaDB-client, while for example mysql_install_db is in the server package. may be just put symlinks into the same package as symlink targets? just like with debs? > +usr/share/man/man1/mysql_plugin.1.gz usr/share/man/man1/mariadb-plugin.1.gz > +usr/share/man/man1/mysql_waitpid.1.gz usr/share/man/man1/mariadb-waitpid.1.gz > +usr/share/man/man1/mysqlaccess.1.gz usr/share/man/man1/mariadb-access.1.gz > +usr/share/man/man1/mysqladmin.1.gz usr/share/man/man1/mariadb-admin.1.gz > +usr/share/man/man1/mysqlanalyze.1.gz usr/share/man/man1/mariadb-analyze.1.gz > +usr/share/man/man1/mysqlbinlog.1.gz usr/share/man/man1/mariadb-binlog.1.gz > +usr/share/man/man1/mysqlcheck.1.gz usr/share/man/man1/mariadb-analyze.1.gz > +usr/share/man/man1/mysqlcheck.1.gz usr/share/man/man1/mariadb-optimize.1.gz > +usr/share/man/man1/mysqlcheck.1.gz usr/share/man/man1/mariadb-repair.1.gz > usr/share/man/man1/mysqlcheck.1.gz usr/share/man/man1/mysqlanalyze.1.gz > usr/share/man/man1/mysqlcheck.1.gz usr/share/man/man1/mysqloptimize.1.gz > usr/share/man/man1/mysqlcheck.1.gz usr/share/man/man1/mysqlrepair.1.gz > +usr/share/man/man1/mysqldump.1.gz usr/share/man/man1/mariadb-dump.1.gz > +usr/share/man/man1/mysqldumpslow.1.gz usr/share/man/man1/mariadb-dump-slow.1.gz > +usr/share/man/man1/mysqlimport.1.gz usr/share/man/man1/mariadb-import.1.gz > +usr/share/man/man1/mysqloptimize.1.gz usr/share/man/man1/mariadb-optimize.1.gz > +usr/share/man/man1/mysqlrepair.1.gz usr/share/man/man1/mariadb-repair.1.gz > +usr/share/man/man1/mysqlreport.1.gz usr/share/man/man1/mariadb-report.1.gz > +usr/share/man/man1/mysqlslap.1.gz usr/share/man/man1/mariadb-slap.1.gz > +usr/share/man/man1/mysqltest.1.gz usr/share/man/man1/mariadb-test.1.gz > diff --git a/debian/mariadb-client-core-10.4.links b/debian/mariadb-client-core-10.4.links > new file mode 100644 > index 00000000000..4c5ff23b38b > --- /dev/null > +++ b/debian/mariadb-client-core-10.4.links > @@ -0,0 +1,4 @@ > +usr/bin/mysql usr/bin/mariadb > +usr/bin/mysqlcheck usr/bin/mariadb-check > +usr/share/man/man1/mysql.1.gz usr/share/man/man1/mariadb.1.gz > +usr/share/man/man1/mysqlcheck.1.gz usr/share/man/man1/mariadb-check.1.gz > diff --git a/debian/mariadb-server-10.4.install b/debian/mariadb-server-10.4.install > index a9cbbfc8a99..8b4db99a421 100644 > --- a/debian/mariadb-server-10.4.install > +++ b/debian/mariadb-server-10.4.install > @@ -87,9 +87,9 @@ usr/share/man/man1/replace.1 > usr/share/man/man1/resolve_stack_dump.1 > usr/share/man/man1/resolveip.1 > usr/share/man/man1/wsrep_sst_common.1 > +usr/share/man/man1/wsrep_sst_mariabackup.1 > usr/share/man/man1/wsrep_sst_mysqldump.1 > usr/share/man/man1/wsrep_sst_rsync.1 > -usr/share/man/man1/wsrep_sst_mariabackup.1 > usr/share/mysql/errmsg-utf8.txt > usr/share/mysql/fill_help_tables.sql > usr/share/mysql/maria_add_gis_sp_bootstrap.sql > diff --git a/debian/mariadb-server-10.4.links b/debian/mariadb-server-10.4.links > new file mode 100644 > index 00000000000..65e0a6650d5 > --- /dev/null > +++ b/debian/mariadb-server-10.4.links > @@ -0,0 +1,18 @@ > +usr/bin/mysql_convert_table_format usr/bin/mariadb-convert-table-format > +usr/bin/mysql_secure_installation usr/bin/mariadb-secure-installation > +usr/bin/mysql_setpermission usr/bin/mariadb-setpermission > +usr/bin/mysql_tzinfo_to_sql usr/bin/mariadb-tzinfo-to-sql > +usr/bin/mysqld_multi usr/bin/mariadb-multi > +usr/bin/mysqld_safe usr/bin/mariadb-safe > +usr/bin/mysqld_safe_helper usr/bin/mariadb-safe-helper > +usr/bin/mysqlhotcopy usr/bin/mariadb-hotcopy > +usr/bin/mysqlshow usr/bin/mariadb-show > +usr/share/man/man1/mysql_convert_table_format.1.gz usr/share/man/man1/mariadb-convert-table-format.1.gz > +usr/share/man/man1/mysql_secure_installation.1.gz usr/share/man/man1/mariadb-secure-installation.1.gz > +usr/share/man/man1/mysql_setpermission.1.gz usr/share/man/man1/mariadb-setpermission.1.gz > +usr/share/man/man1/mysql_tzinfo_to_sql.1.gz usr/share/man/man1/mariadb-tzinfo-to-sql.1.gz > +usr/share/man/man1/mysqld_multi.1.gz usr/share/man/man1/mariadb-multi.1.gz > +usr/share/man/man1/mysqld_safe.1.gz usr/share/man/man1/mariadb-safe.1.gz > +usr/share/man/man1/mysqld_safe_helper.1.gz usr/share/man/man1/mariadb-safe-helper.1.gz > +usr/share/man/man1/mysqlhotcopy.1.gz usr/share/man/man1/mariadb-hotcopy.1.gz > +usr/share/man/man1/mysqlshow.1.gz usr/share/man/man1/mariadb-show.1.gz > diff --git a/debian/mariadb-server-core-10.4.links b/debian/mariadb-server-core-10.4.links > new file mode 100644 > index 00000000000..90ed9ee01cf > --- /dev/null > +++ b/debian/mariadb-server-core-10.4.links > @@ -0,0 +1,4 @@ > +usr/bin/mysql_install_db usr/bin/mariadb-install-db > +usr/bin/mysql_upgrade usr/bin/mariadb-upgrade > +usr/share/man/man1/mysql_install_db.1.gz usr/share/man/man1/mariadb-install-db.1.gz > +usr/share/man/man1/mysql_upgrade.1.gz usr/share/man/man1/mariadb-upgrade-db.1.gz > diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt > index 3d31cda4ceb..7be46ac1985 100644 > --- a/scripts/CMakeLists.txt > +++ b/scripts/CMakeLists.txt > @@ -332,6 +332,7 @@ ELSE() > COMPONENT ${${file}_COMPONENT} > ) > ENDFOREACH() > + > SET (wsrep_sst_rsync_wan ${CMAKE_CURRENT_BINARY_DIR}/wsrep_sst_rsync_wan) > ADD_CUSTOM_COMMAND( > OUTPUT ${wsrep_sst_rsync_wan} > Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

2 1

[Maria-developers] GSoC [MDEV-5092]
by Мирослав Коберский 14 Jun '19

14 Jun '19

Hello, I'm working in implementing UPDATE .. RETURNING. For beginning i'm working on single-table update. I've made parser to parse RETURNING clause in UPDATE. There was a problem that columns that should be updated and columns that should be returned were stored in the same list in lex named item_list. I'm trying now to divide them. Not sure that swapping lists two times during parsing between each other is the best idea. I've implemented primitive returning of result set every time doing an update (even without RETURNING clause) to understand how client understands such response. It successfully returns result set in very simple updates. https://github.com/Mup0c/server/commits/MDEV-5092 -- Miroslav Koberskii

2 1

Re: [Maria-developers] 030c299f76f: MDEV-17591 Create MariaDB named commands/symlinks
by Sergei Golubchik 13 Jun '19

13 Jun '19

Hi, Rasmus! On Jun 11, Rasmus Johansson wrote: > revision-id: 030c299f76f (mariadb-10.4.5-33-g030c299f76f) > parent(s): 973b281e599 > author: Rasmus Johansson <razze(a)iki.fi> > committer: Rasmus Johansson <razze(a)iki.fi> Is that an address you want to have in git history? You already have two, just making sure the third one is intentional. > timestamp: 2019-06-09 07:33:41 +0000 > message: > > MDEV-17591 Create MariaDB named commands/symlinks > > diff --git a/cmake/symlinks.cmake b/cmake/symlinks.cmake > new file mode 100644 > index 00000000000..da38d2c28d5 > --- /dev/null > +++ b/cmake/symlinks.cmake > @@ -0,0 +1,8 @@ > +# Add MariaDB symlinks > +MACRO(CREATE_MARIADB_SYMLINK binaryname symlinkname) > + IF (EXISTS ${CMAKE_CURRENT_BINARY_DIR}/${binaryname} AND NOT WIN32) > + install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink ${binaryname} ${CMAKE_CURRENT_BINARY_DIR}/${symlinkname})" COMPONENT symlinks) > + install(CODE "message(\"-- Created symlink: ${symlinkname} -> ${binaryname}\")" COMPONENT symlinks) > + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${symlinkname} DESTINATION ${INSTALL_BINDIR} COMPONENT symlinks) > + ENDIF() > +ENDMACRO(CREATE_MARIADB_SYMLINK) Hmm, that's very weird. 1. you test whether a binary exists _at configure time_ but the binary is only created during the build. So, the test is always false, unless you re-run cmake after the build, which is not a normal build procedure. 2. you create a symlink _at install time_ but in a CMAKE_CURRENT_BINARY_DIR. This is strange decision. Normally, build step creates files in CMAKE_CURRENT_BINARY_DIR, and install step creates files in INSTALL_BINDIR. Basically, to fix that you should either create symlinks as a new build target that dependents on binaries being created first. And then install these symlinks normally. Or create symlinks during install time directly at the correct install location, not in the binary dir. > diff --git a/debian/mariadb-client-10.4.links b/debian/mariadb-client-10.4.links > index 4a504969246..b93d2d7f8fd 100644 > --- a/debian/mariadb-client-10.4.links > +++ b/debian/mariadb-client-10.4.links > @@ -1,6 +1,44 @@ > +usr/bin/mysql_find_rows usr/bin/mariadb-find-rows > +usr/bin/mysql_fix_extensions usr/bin/mariadb-fix-extenstions > +usr/bin/mysql_plugin usr/bin/mariadb-plugin Why there's a separate rpm with symlinks, but no separate deb with symlinks (they're direcly in the client deb) ? Why would you want different approaches for deb and rpm? > +usr/bin/mysqld_multi usr/bin/mariadb-multi > +usr/bin/mysqld_safe usr/bin/mariadb-safe > +usr/bin/mysqld_safe_helper usr/bin/mariadb-safe-helper Also you seem to symlink a lot more files in deb than in rpm or in tar.gz. Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

2 1

Re: [Maria-developers] 66f931bcd69: MDEV-18727 improve DML operation of System Versioning
by Sergei Golubchik 12 Jun '19

12 Jun '19

Hi, Aleksey! Looks ok, thanks! No requests to change anything, but still a couple of "why" questions, see below On Jun 07, Aleksey Midenkov wrote: > revision-id: 66f931bcd69 (mariadb-10.3.12-100-g66f931bcd69) > parent(s): f4484dfdbf2 > author: Aleksey Midenkov <midenok(a)gmail.com> > committer: Aleksey Midenkov <midenok(a)gmail.com> > timestamp: 2019-03-26 15:04:52 +0300 > message: > > MDEV-18727 improve DML operation of System Versioning > MDEV-18957 UPDATE with LIMIT clause is wrong for versioned partitioned tables > > UPDATE, DELETE: replace linear search of current/historical records > with vers_setup_conds(). > > diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc > index b8ed887b52d..a00cc2a8fb0 100644 > --- a/sql/sql_parse.cc > +++ b/sql/sql_parse.cc > @@ -4975,8 +4975,10 @@ mysql_execute_command(THD *thd) > { > result= new (thd->mem_root) multi_delete(thd, aux_tables, > lex->table_count); > - if (unlikely(result)) > + if (likely(result)) heh, good catch > { > + if (unlikely(select_lex->vers_setup_conds(thd, aux_tables))) > + goto multi_delete_error; > res= mysql_select(thd, > select_lex->get_table_list(), > select_lex->with_wild, > diff --git a/sql/sql_select.cc b/sql/sql_select.cc > index 6915d4d23ca..28b54e0b3b1 100644 > --- a/sql/sql_select.cc > +++ b/sql/sql_select.cc > @@ -779,20 +781,35 @@ int SELECT_LEX::vers_setup_conds(THD *thd, TABLE_LIST *tables) > } > } > > + bool is_select= false; > + switch (thd->lex->sql_command) > + { > + case SQLCOM_SELECT: > + case SQLCOM_INSERT_SELECT: > + case SQLCOM_REPLACE_SELECT: > + case SQLCOM_DELETE_MULTI: > + case SQLCOM_UPDATE_MULTI: > + is_select= true; > + default: > + break; > + } > + > for (table= tables; table; table= table->next_local) > { > - if (!table->table || !table->table->versioned()) > + if (!table->table || table->is_view() || !table->table->versioned()) why? > continue; > > vers_select_conds_t &vers_conditions= table->vers_conditions; > > #ifdef WITH_PARTITION_STORAGE_ENGINE > - /* > - if the history is stored in partitions, then partitions > - themselves are not versioned > - */ > - if (table->partition_names && table->table->part_info->vers_info) > + Vers_part_info *vers_info; > + if (is_select && table->table->part_info && > + (vers_info= table->table->part_info->vers_info)) > + { > + if (table->partition_names) Why is that? You don't seem to use vers_info anywhere. Nor I see a reason for two nested if()'s intead of one, as before (with an additional `is_select &&` condition) > { > + /* If the history is stored in partitions, then partitions > + themselves are not versioned. */ > if (vers_conditions.is_set()) > { > my_error(ER_VERS_QUERY_IN_PARTITION, MYF(0), table->alias.str); Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

2 1

Re: [Maria-developers] [Commits] 5bfc8128c83: MDEV-8874 Replication filters configured in my.cnf are ignored if slave reset and reconfigured
by andrei.elkin＠pp.inet.fi 12 Jun '19

12 Jun '19

Sachin, hello. The patch is alright. Thanks for a good piece of work! Andrei > revision-id: 5bfc8128c8381e9347589aa0e27cd4fbd66ef19d (mariadb-10.1.39-57-g5bfc8128c83) > parent(s): 2e0fb93e6f3359ffe146f309e32c59922400b7c4 > author: Sachin > committer: Sachin > timestamp: 2019-06-12 15:58:44 +0530 > message: > > MDEV-8874 Replication filters configured in my.cnf are ignored if slave reset and reconfigured > > Don't delete the rpl_filter on RESET SLAVE. >

1 0

Re: [Maria-developers] [Commits] 321a43d9ac5: MDEV-8874 Replication filters configured in my.cnf are ignored if slave reset and reconfigured
by andrei.elkin＠pp.inet.fi 12 Jun '19

12 Jun '19

Sachin, salute! The patch looks good, but I think it misses some verbosity in few places, please read on. > revision-id: 321a43d9ac586034790407dc519c72e90b3bff81 (mariadb-10.1.39-54-g321a43d9ac5) > parent(s): b003b0c934cf6b59358e31144c4f69cf34622fb8 > author: Sachin > committer: Sachin > timestamp: 2019-06-06 12:03:45 +0530 > message: > > MDEV-8874 Replication filters configured in my.cnf are ignored if slave reset and reconfigured > > Don't delete the rpl_filter on RESET SLAVE. > > --- > mysql-test/lib/My/Config.pm | 8 +- > mysql-test/suite/multi_source/mdev-8874.cnf | 25 +++++ > mysql-test/suite/multi_source/mdev-8874.result | 93 +++++++++++++++++++ > mysql-test/suite/multi_source/mdev-8874.test | 123 +++++++++++++++++++++++++ > sql/rpl_mi.cc | 2 - > 5 files changed, 248 insertions(+), 3 deletions(-) > > diff --git a/mysql-test/lib/My/Config.pm b/mysql-test/lib/My/Config.pm > index 12647edf0a4..349a397ed6d 100644 > --- a/mysql-test/lib/My/Config.pm > +++ b/mysql-test/lib/My/Config.pm > @@ -327,7 +327,13 @@ sub new { > # Skip comment > next; > } > - > + > + # Replication Filter > + elsif ( $line =~ /^([\w]+.[\w]+)\s*=\s*(.*)\s*/){ > + my $option= $1; > + my $value= $2; > + $self->insert($group_name, $option, $value); > + } What was a problem at the above point? [Anything that you think may not be straightforward for a patch reader is always worth to highlight in the description.] > else { > croak "Unexpected line '$line' found in '$path'"; > } > diff --git a/mysql-test/suite/multi_source/mdev-8874.cnf b/mysql-test/suite/multi_source/mdev-8874.cnf > new file mode 100644 > index 00000000000..01da70cbaa0 > --- /dev/null > +++ b/mysql-test/suite/multi_source/mdev-8874.cnf > @@ -0,0 +1,25 @@ > +!include my.cnf > + > +[mysqld.1] > +log-bin > +log-slave-updates > + > +[mysqld.2] > +log-bin > +log-slave-updates > + > +[mysqld.3] > +log-bin > +log-slave-updates > + > +[mysqld.4] > +server-id=4 > +log-bin=server4-bin > +log-slave-updates > +m1.replicate_ignore_table='a.t1' > +m2.replicate_ignore_table='b.t1' > +replicate_ignore_table='c.t1' > + > +[ENV] > +SERVER_MYPORT_4= @mysqld.4.port > +SERVER_MYSOCK_4= @mysqld.4.socket > diff --git a/mysql-test/suite/multi_source/mdev-8874.result b/mysql-test/suite/multi_source/mdev-8874.result > new file mode 100644 > index 00000000000..6fb364a3d2d > --- /dev/null > +++ b/mysql-test/suite/multi_source/mdev-8874.result > @@ -0,0 +1,93 @@ > +create database a; > +use a; > +create table t1(a int); > +insert into t1 values(1); > +create table t2(a int); > +insert into t2 values(1); > +create database b; > +use b; > +create table t1(a int); > +insert into t1 values(1); > +create table t2(a int); > +insert into t2 values(1); > +create database c; > +use c; > +create table t1(a int); > +insert into t1 values(1); > +create table t2(a int); > +insert into t2 values(1); > +change master 'm1' to master_port=MYPORT_1 , master_host='127.0.0.1', master_user='root'; > +change master 'm2' to master_port=MYPORT_2 , master_host='127.0.0.1', master_user='root'; > +change master to master_port=MYPORT_3 , master_host='127.0.0.1', master_user='root'; > +start all slaves; > +set default_master_connection = 'm1'; > +include/wait_for_slave_to_start.inc > +set default_master_connection = 'm2'; > +include/wait_for_slave_to_start.inc > +set default_master_connection = ''; > +include/wait_for_slave_to_start.inc > +use a; > +show tables; > +Tables_in_a > +t2 > +use b; > +show tables; > +Tables_in_b > +t2 > +use c; > +show tables; > +Tables_in_c > +t2 > +#TEST > +STOP ALL SLAVES; > +Warnings: > +Note 1938 SLAVE 'm2' stopped > +Note 1938 SLAVE '' stopped > +Note 1938 SLAVE 'm1' stopped > +RESET SLAVE 'm1' ALL ; > +RESET SLAVE 'm2' ALL ; > +RESET SLAVE ALL ; > +drop database a; > +drop database b; > +drop database c; > +change master 'm1' to master_port=MYPORT_1 , master_host='127.0.0.1', master_user='root'; > +change master 'm2' to master_port=MYPORT_2 , master_host='127.0.0.1', master_user='root'; > +change master to master_port=MYPORT_3 , master_host='127.0.0.1', master_user='root'; > +start all slaves; > +Warnings: > +Note 1937 SLAVE 'm2' started > +Note 1937 SLAVE '' started > +Note 1937 SLAVE 'm1' started > +set default_master_connection = 'm1'; > +include/wait_for_slave_to_start.inc > +set default_master_connection = 'm2'; > +include/wait_for_slave_to_start.inc > +set default_master_connection = ''; > +include/wait_for_slave_to_start.inc > +use a; > +show tables; > +Tables_in_a > +t2 > +use b; > +show tables; > +Tables_in_b > +t2 > +use c; > +show tables; > +Tables_in_c > +t2 > +#CleanUp > +drop database a; > +drop database b; > +drop database c; > +stop all slaves; > +Warnings: > +Note 1938 SLAVE 'm2' stopped > +Note 1938 SLAVE '' stopped > +Note 1938 SLAVE 'm1' stopped > +SET default_master_connection = "m1"; > +include/wait_for_slave_to_stop.inc > +SET default_master_connection = "m2"; > +include/wait_for_slave_to_stop.inc > +SET default_master_connection = ""; > +include/wait_for_slave_to_stop.inc To the test part, I suggest we always follow a well-know description pattern (which is a great time saver when it comes to future test maintenance, apart from the current reviewing), to include the task id (present in the name though) and title, what failed and *how* the test proves the fixes. I can bet the failure was also exposed by reset @@global.'connection_name'.'rule_type' variables, which the test is really missing. Could you please add up that? > diff --git a/mysql-test/suite/multi_source/mdev-8874.test b/mysql-test/suite/multi_source/mdev-8874.test > new file mode 100644 > index 00000000000..b6d1aafe139 > --- /dev/null > +++ b/mysql-test/suite/multi_source/mdev-8874.test > @@ -0,0 +1,123 @@ > +--source include/not_embedded.inc > +--source include/have_innodb.inc > +--source include/have_debug.inc > + > +--connect (server_1,127.0.0.1,root,,,$SERVER_MYPORT_1) > +--connect (server_2,127.0.0.1,root,,,$SERVER_MYPORT_2) > +--connect (server_3,127.0.0.1,root,,,$SERVER_MYPORT_3) > +--connect (server_4,127.0.0.1,root,,,$SERVER_MYPORT_4) > + > +--connection server_1 > +create database a; > +use a; > +create table t1(a int); > +insert into t1 values(1); > +create table t2(a int); > +insert into t2 values(1); > +--save_master_pos > + > +--connection server_2 > +create database b; > +use b; > +create table t1(a int); > +insert into t1 values(1); > +create table t2(a int); > +insert into t2 values(1); > +--save_master_pos > + > +--connection server_3 > +create database c; > +use c; > +create table t1(a int); > +insert into t1 values(1); > +create table t2(a int); > +insert into t2 values(1); > +--save_master_pos > + > +--connection server_4 > +--disable_warnings > +--replace_result $SERVER_MYPORT_1 MYPORT_1 > +eval change master 'm1' to master_port=$SERVER_MYPORT_1 , master_host='127.0.0.1', master_user='root'; > +--replace_result $SERVER_MYPORT_2 MYPORT_2 > +eval change master 'm2' to master_port=$SERVER_MYPORT_2 , master_host='127.0.0.1', master_user='root'; > +--replace_result $SERVER_MYPORT_3 MYPORT_3 > +eval change master to master_port=$SERVER_MYPORT_3 , master_host='127.0.0.1', master_user='root'; > +start all slaves; > +set default_master_connection = 'm1'; > +--source include/wait_for_slave_to_start.inc > +set default_master_connection = 'm2'; > +--source include/wait_for_slave_to_start.inc > +set default_master_connection = ''; > +--source include/wait_for_slave_to_start.inc > + > +--enable_warnings > +--sync_with_master 0,'m1' > +--sync_with_master 0,'m2' > +--sync_with_master 0,'' > +use a; When the test proves something via printing out, it's always good to prepare the output with an --echo what-is-expected so a thankful reader will bless you .. and the others may mercy :-) > +show tables; > +use b; > +show tables; > +use c; > +show tables; > +--echo #TEST > +STOP ALL SLAVES; > +RESET SLAVE 'm1' ALL ; > +RESET SLAVE 'm2' ALL ; > +RESET SLAVE ALL ; > +drop database a; > +drop database b; > +drop database c; > +--replace_result $SERVER_MYPORT_1 MYPORT_1 > +eval change master 'm1' to master_port=$SERVER_MYPORT_1 , master_host='127.0.0.1', master_user='root'; > +--replace_result $SERVER_MYPORT_2 MYPORT_2 > +eval change master 'm2' to master_port=$SERVER_MYPORT_2 , master_host='127.0.0.1', master_user='root'; > +--replace_result $SERVER_MYPORT_3 MYPORT_3 > +eval change master to master_port=$SERVER_MYPORT_3 , master_host='127.0.0.1', master_user='root'; > +start all slaves; > +set default_master_connection = 'm1'; > +--source include/wait_for_slave_to_start.inc > +set default_master_connection = 'm2'; > +--source include/wait_for_slave_to_start.inc > +set default_master_connection = ''; > +--source include/wait_for_slave_to_start.inc > +--sync_with_master 0,'m1' > +--sync_with_master 0,'m2' > +--sync_with_master 0,'' > + Naturally, the same applies to the following block. > +use a; > +show tables; > +use b; > +show tables; > +use c; > +show tables; > + > + > +#--echo #restart the server > +#--source include/restart_mysqld.inc > + > + > +--echo #CleanUp > +--connection server_1 > +drop database a; > +--save_master_pos > + > +--connection server_2 > +drop database b; > +--save_master_pos > + > +--connection server_3 > +drop database c; > +--save_master_pos > + > +--connection server_4 > +--sync_with_master 0,'m1' > +--sync_with_master 0,'m2' > +--sync_with_master 0,'' > +stop all slaves; > +SET default_master_connection = "m1"; > +--source include/wait_for_slave_to_stop.inc > +SET default_master_connection = "m2"; > +--source include/wait_for_slave_to_stop.inc > +SET default_master_connection = ""; > +--source include/wait_for_slave_to_stop.inc > diff --git a/sql/rpl_mi.cc b/sql/rpl_mi.cc > index 70e60b1d4ad..7aea89337a7 100644 > --- a/sql/rpl_mi.cc > +++ b/sql/rpl_mi.cc > @@ -122,8 +122,6 @@ Master_info::~Master_info() > */ > if (strncmp(connection_name.str, STRING_WITH_LEN("wsrep"))) > #endif > - rpl_filters.delete_element(connection_name.str, connection_name.length, > - (void (*)(const char*, uchar*)) free_rpl_filter); > my_free(connection_name.str); > delete_dynamic(&ignore_server_ids); > mysql_mutex_destroy(&run_lock); > _______________________________________________ > commits mailing list > commits(a)mariadb.org > https://lists.askmonty.org/cgi-bin/mailman/listinfo/commits

2 1

[Maria-developers] GSoC [MDEV-6017]
by Alexey Mogilyovkin 10 Jun '19

10 Jun '19

Hello. This is the thread where I would like to publish my weekly reports every Monday. During preparatory weeks I set up development environment. I briefly examined testing framework, debugging process and now I am ready to start work. Work progress will be available on my github repository in branch "MDEV-6017" https://github.com/ZeroICQ/server/tree/MDEV-6017

3 5

[Maria-developers] Weekly Report
by Rucha Deodhar 10 Jun '19

10 Jun '19

Report for week 2: Hello! This week I ran the test case for parser written in last week and made changes in the parser according to the suggestions given, tried making insert...select..returning work. I analysed how RETURNING in delete is implemented, how insert is implemented, and how the data can be sent, which functions are responsible and what those functions do, to carry out various steps and what these steps are in order to send the data to the client. I came up with a plan where code for RETURNING should go and files where the code has to be written. There were few things which weren't very clear initially. So I referred to some documents on the internet and some source code document. My github repo is up to date with the changes in the code, result and test files. Regards, Rucha

1 0

[Maria-developers] MDEV-17951 mariadb symlinks
by Rasmus Johansson 09 Jun '19

09 Jun '19

Hi Serg, I've addressed general review requests by Wlad and review requests by Otto related to Debian packaging. The latest commit is here: https://github.com/MariaDB/server/commit/030c299f76fac044d43325abb02b5751fe… Could you give it final review to see if it can be pushed to main 10.4? Rasmus -- Rasmus Johansson, VP Engineering MariaDB | t: +358 50 499 9589 | Skype: ratzpo

1 0

Re: [Maria-developers] a2520676ba1: MDEV-16249 CHECKSUM TABLE for a spider table is not parallel and saves all data in memory in the spider head by default
by Sergei Golubchik 06 Jun '19

06 Jun '19

Hi, Kentoku! On Jun 05, Kentoku wrote: > revision-id: a2520676ba1 (mariadb-10.4.5-10-ga2520676ba1) > parent(s): 31fe70290c5 > author: Kentoku <kentokushiba(a)gmail.com> > committer: Kentoku <kentokushiba(a)gmail.com> > timestamp: 2019-05-23 14:36:18 +0900 > message: > > MDEV-16249 CHECKSUM TABLE for a spider table is not parallel and saves all data in memory in the spider head by default please remember to add an empty line after the first line in the commit comment, see https://git-scm.com/docs/git-commit#_discussion > add checksum_null for setting null value of checksum > > diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc > index 4e7fccca3af..e47da94ab20 100644 > --- a/sql/ha_partition.cc > +++ b/sql/ha_partition.cc > @@ -8213,7 +8214,11 @@ int ha_partition::info(uint flag) > stats.delete_length+= file->stats.delete_length; > if (file->stats.check_time > stats.check_time) > stats.check_time= file->stats.check_time; > - stats.checksum+= file->stats.checksum; > + if (!file->stats.checksum_null) > + { > + stats.checksum+= file->stats.checksum; > + stats.checksum_null= FALSE; > + } I'm not sure about it. When a checksum can be NULL? I mean, normally, if you do A+B+C and one of the values is NULL, then the whole sum is NULL. But here you simply skip NULLs, like in aggregate SUM() function. To understand what semantics works better in this case, I need to understand when a checksum can be NULL in spider. > } > if (stats.records && stats.records < 2 && > !(m_file[0]->ha_table_flags() & HA_STATS_RECORDS_IS_EXACT)) All other server changes look good, thanks. Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

2 2

Re: [Maria-developers] Fwd: [Commits] a602998: MDEV-19580 Unrelated JOINs corrupt usage of 'WHERE function() IN (subquery)'
by Sergey Petrunia 05 Jun '19

05 Jun '19

Hello Igor, On Sat, Jun 01, 2019 at 01:16:04PM -0700, Igor Babaev wrote: > Please review this patch for the bug that we discussed on slack a couple of > days ago. > As you can see the extra calls of make_cond_for_table() with > used_table=RAND_TABLE_BIT were not enough (it can be seen in debugger why). > It took me quite a lot of time to figure out that I could not trust > Item::marker==2 to check whether a conjunct was previously attached > or not. > First, I've found a testcase that stopped working after the patch. (As far as I understand, we should pass RAND_TABLE_BIT when constructing JOIN::outer_ref_cond): create table ten(a int); insert into ten values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); create table t1 (a int, b int, c int); insert into t1 select a,a,a from ten; create table t2 as select a,b,c from t1; mysql> explain format=json select (select max(c) from t1 where t1.a=t2.a and rand()=t2.b) from t2\G *************************** 1. row *************************** EXPLAIN: { "query_block": { "select_id": 1, "table": { "table_name": "t2", "access_type": "ALL", "rows": 10, "filtered": 100 }, "subqueries": [ { "query_block": { "select_id": 2, "table": { "table_name": "t1", "access_type": "ALL", "rows": 10, "filtered": 100, "attached_condition": "((t1.a = t2.a) and (rand() = t2.b))" } } } ] } } 1 row in set (0.00 sec) If I run the select, it hits a breakpoint in Item_func_rand::val_real. Now, after this patch (I applied it to 10.1): mysql> explain format=json select (select max(c) from t1 where t1.a=t2.a and rand()=t2.b) from t2\G *************************** 1. row *************************** EXPLAIN: { "query_block": { "select_id": 1, "table": { "table_name": "t2", "access_type": "ALL", "rows": 10, "filtered": 100 }, "subqueries": [ { "query_block": { "select_id": 2, "table": { "table_name": "t1", "access_type": "ALL", "rows": 10, "filtered": 100, "attached_condition": "(t1.a = t2.a)" } } } ] } } and running the SELECT doesn't hit the breakpoint in Item_func_rand::val_real A bit more of cosmetic feedback below, marked with 'psergey:' > > @@ -8968,6 +8963,20 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND > *cond) > { > tmp= make_cond_for_table(thd, cond, used_tables, current_map, i, > FALSE, FALSE); > + if (tab == join->join_tab + last_top_base_tab_idx) > + { > + /* > + This pushes conjunctive conditions of WHERE condition such > that: > + - their used_tables() contain RAND_TABLE_BIT > + - the conditions does not refer to any fields > + (such like rand() > 0.5) > + */ > + table_map rand_table_bit= (table_map) RAND_TABLE_BIT; psergey: what is the purpose of rand_table_bit? If we can't use RAND_TABLE_BIT where a table bitmap is expected, we should fix that. The way it's done now it looks redundant and is confusing. (this also applies to other declarations if rand_table_bit) > + COND *rand_cond= make_cond_for_table(thd, cond, used_tables, > + rand_table_bit, -1, > + FALSE, FALSE); > + add_cond_and_fix(thd, &tmp, rand_cond); > + } > } > /* Add conditions added by add_not_null_conds(). */ > if (tab->select_cond) ... > @@ -18852,11 +18880,27 @@ make_cond_for_table_from_pred(THD *thd, Item > *root_cond, Item *cond, > Item *item; > while ((item=li++)) > { > + /* > + Special handling of top level conjuncts with RAND_TABLE_BIT: > + if such a conjunct contains a reference to a field then it is > pushed > + to the corresponding table by the same rule as all other > conjuncts. > + Otherwise, if the conjunct is used in WHERE is is pushed to the > last > + joined table, if is it is used in ON condition of an outer join > it > + is pushed into the last inner table of the outer join. Such > conjuncts > + are pushed in a call of make_cond_for_table_from_pred() with the > + parameter 'used_table' equal to RAND_TABLE_BIT. > + */ > + if (is_top_and_level && used_table == rand_table_bit && > + item->used_tables() != rand_table_bit) > + { > + /* The conjunct with RAND_TABLE_BIT has been allready pushed */ psergey: typo: allready > + continue; > + } > Item *fix=make_cond_for_table_from_pred(thd, root_cond, item, > tables, used_table, > - join_tab_idx_arg, > + join_tab_idx_arg, > exclude_expensive_cond, > - retain_ref_cond); > + retain_ref_cond, false); > if (fix) > new_cond->argument_list()->push_back(fix); BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

[Maria-developers] Weekly Report
by Rucha Deodhar 03 Jun '19

03 Jun '19

Report for week 1: Hello! I made test cases and extended the parser for: INSERT...RETURNING * and INSERT...RETURNING <col_name/names> It is working fine for INSERT...ON DUPLICATE KEY UPDATE, INSERT...IGNORE, INSERT...SET, and simple insert statements. As I have never coded parser before, it was throwing errors during build and was also crashing the server sometimes. I managed to fix the errors eventually. Initially, I tried to extend <column_names> in RETURNING by referring how column names are parsed in INSERT statement, like using opt_fields production for insert_field_list, modified it a little and tried to see it if that would work. But it didn't. So I referred to DELETE...RETURNING as it was suggested earlier. I am still trying to extend it for INSERT...SELECT. It is not showing syntax error but shows error "column count doesn't match value count at row 1." I will add the test cases for this once it is working. Regards, Rucha

2 1

Re: [Maria-developers] [Commits] b3473961810: MDEV-11094: Blackhole table updates on slave fail when row annotation is enabled
by Sergey Vojtovich 29 May '19

29 May '19

Hi Sujatha, On Wed, May 29, 2019 at 04:37:27PM +0530, sujatha wrote: > revision-id: b347396181018cedc946450cb49891f1a0aa4575 (mariadb-10.1.39-48-gb3473961810) > parent(s): 8358c6f03edb941be488f009c2bd0eb9df47e8c5 > author: Sujatha > committer: Sujatha > timestamp: 2019-05-29 15:18:52 +0530 > message: > > MDEV-11094: Blackhole table updates on slave fail when row annotation is enabled ...skip... > diff --git a/storage/blackhole/ha_blackhole.cc b/storage/blackhole/ha_blackhole.cc > index 01aaa9ea15f..43bcdc541a1 100644 > --- a/storage/blackhole/ha_blackhole.cc > +++ b/storage/blackhole/ha_blackhole.cc > @@ -25,6 +25,16 @@ > #include "ha_blackhole.h" > #include "sql_class.h" // THD, SYSTEM_THREAD_SLAVE_SQL > > +static bool is_row_based_replication(THD *thd) > +{ > + /* > + A row event which has its thd->query() == NULL or a row event which has > + replicate_annotate_row_events enabled. In the later case the thd->query() > + will be pointing to the query, received through replicated annotate event > + from master. > + */ > + return ((thd->query() == NULL) || thd->variables.binlog_annotate_row_events); Any reason to have so many redundant parenthesis? Why code comment rather than function comment? May be move "thd->system_thread == SYSTEM_THREAD_SLAVE_SQL" inside as well? Regards, Sergey

1 0

[Maria-developers] Proposal for automatic code formatting (clang-format)
by Eugene Kosov 27 May '19

27 May '19

Hi. I'm proposing to add some tool that automatically formats source code. Main purpose is overall productivity. It's possible to save time on manual formatting. Also, reading a uniformly formatted code is easier. Some new languages has one correct coding style and code formatters. Examples are Go (https://golang.org/cmd/gofmt/) and (to be done) Rust (https://github.com/rust-dev-tools/fmt-rfcs/blob/master/guide/guide.md) I suggest to read 'Motivation - why use a formatting tool' from second link. There is no single correct coding style in C and C++ worlds: different projects use different styles if any at all. But there is a tool which does formatting and supports different styles: clang-format. It's a binary available for every major platform. It's configurable: just put .clang-format file with formatting options in source directory and formatter will use it for current and nested directories. I actually do not propose here to use exactly clang-format but as I know it's the best of its kind. Here is a demo https://youtu.be/uZI_Qla4pNA?t=1893 Tool is configurable but it will not act exactly as you might expect and this is a pay for uniformity. So, let's add .clang-format to root directory, start using and polishing style. Here is some info on how to add it to some text editors https://clang.llvm.org/docs/ClangFormat.html I do not(!) suggesting to reformat whole project. At least for now. Here is my PR to add .clang-format files to server and InnoDB: https://github.com/MariaDB/server/pull/1270 I think that eventually .clang-format usage will lead to reformatting everything and having some project-wide precommit hook which will properly format your diffs. Do you have any objections or questions on having such a tool? Let's discuss, please. -- Eugene

1 0

Re: [Maria-developers] Regarding approach used for Early NULL filtering in the range optimizer(MDEV-15777)
by Sergey Petrunia 27 May '19

27 May '19

On Fri, May 24, 2019 at 04:39:51PM -0700, Igor Babaev wrote: > On 05/24/2019 01:41 PM, Sergey Petrunia wrote: > > On Fri, May 24, 2019 at 10:08:04AM -0700, Igor Babaev wrote: > > > On 05/24/2019 04:03 AM, Varun Gupta wrote: > > > > Hi Igor, > > > > After discussing with Sergey, we came up with these conclusions as to > > > > why we used the approach of going through all the keyuses in the KEYUSE > > > > array > > > > > > > > Cases to consider: > > > > we have an index on column a > > > > > > > > 1) a OP const > > > > where OP can be (</<=/=/between/ etc on which we do range analysis) . > > > > This case is already handled by the range optimizer. We do not create a > > > > NULL rejecting predicate for such conditions. > > > > > > > > 2) eq_join conditions (tbl1.a = tbl2.col2) > > > > This is the specific case we have tried to implement in MDEV-15777, we > > > > create NULL rejecting predicates for the keyuses for each table and then > > > > feed these predicates to range optimizer > > > There is no logical explanation for checking null rejection of fields used > > > in equalities with the help of array keyuse. > > The logic here is as follows: > > > > We already collect the set of KEYUSE::null_rejecting attributes. Its > > meaning is very close to what the patch needs. > The meaning of this argument is to be used a key value for ref access by an > index i1. > You need range access by a different index i2. There is no logical > connection > between these two indexes. The usage of i1 can be blocked by IGNORE clause > and this case > there won't be any keyuses for it. But the equality from which the > null-rejection can be > deduced is still there. You need range scan to reduce the number of records > produced > by the left operand of a join operation. And you should not care weather it > should be the > argument of any ref access. > Here is the situation: > The field of interest for range access is t1.a and you have two conditions > t1.a=t2.a with index i1 on t2.a > t1.b=t3.b with index i2 on t3.b > The use prohibited to use i1. t1.a apparently remains null-rejecting. Yet it > cannot be not detected > as such by a look through array of keyuses. I don't follow the logic. What MDEV-15777 does is to basically look at KEYUSE object, which represent tbl.key_col = other_column and feed this into the range optimizer: tbl.key_col IS NOT NULL If the query CAN use a certain index: - we will have a KEYUSE object for it - we will feed the IS NOT NULL into the range optimizer. if the query CANNOT use a certain index (because of IGNORE INDEX), then - we won't have KEYUSE object for that index (and so won't construct an IS NOT NULL condition for it) - range optimizer will not build ranges for it (so it doesn't really matter if IS NOT NULL was constructed or not) Can you provide a full example? > > > > Because of this, the part of the patch for MDEV-15777 which analyzes the KEYUSE > > array is very small: 122 lines including the comment (I counted > > make_null_rejecting_conds() and add_cond()). > > > > I think, the CPU overhead is small, too. > > > > > > 3) a < func(b,c) > > > > we do not handle this case, because: > > > > > > > > a) It is harder to process. We would need to process the whole WHERE > > > > clause to infer > > > > non-nullability of the condition. Example: > > > > > > > > (t1.f2+1 < t1.f1+t1.f3) OR ... OR t1.f1 is null > > > > > > > > here the left part of the OR condition doesn't allow f1 to be NULL, but > > > > the > > > > right condition allows it. We would also need to take into account tables > > > > inside outer joins. We would need to go through Item classes and add code > > > > which say "Item_xxx() will compute to false when its argument Y is null" > > > > (there is item->not_null_tables() currently, but not not_null_columns()). > > > Have you actually looked at the implementations of the virtual function > > > not_null_tables()? > > > Do you need me to provide the implementation of the virtual function > > > not_null_columns()? > > It will require multiple implementations. At the moment we have: > > > > nm mysqld --demangle | grep 'Item.*::not_null_tables' | wc -l > > 21 > > > > Another question: do you intend to collect not_null_columns() for all columns, > > or just columns that are a part of some index? > > > > What would a good data structure to store a set of tablename.column_name ? > It will be just a bitmap created for each table that can be used for many > other purposes. > If you want to filter out those that are not part of any index it can be > easily done > (even once for any table share) My point was that constructing such a bitmap is a lot heavier than the not_null_table_map(). table.h declares a field bitmap class: typedef Bitmap<MAX_FIELDS> Field_map; but it's huge: (gdb) p sizeof(Field_map) $52 = 544 and note that we will not be able to just put one Field_map into the TABLE object: If we think about how to compute not_null_columns() for Item_cond_and or Item_cond_or: - Item_cond_and will compute a union of all its arguments' not_null_columns(). - Item_cond_or will compute an intersection of all its arguments' not_null_columns(). - Other items have different logic. that is, we will need to keep multiple "set of fields in tables" objects around. Just one bitmap attached to the table will not suffice. This is doable, but it's going to be more code and have more CPU overhead. And I'm still not convinced why we should do this. It is not clear if this will fit the scope of MDEV-15777 and once we got it done, what's the use of it? Is this something other query optimizers do (so the users expect it?)? > > > > > > b) the conditions in form of arbitrary functions are not as frequently > > > > used as > > > > ref access conditions. > > > > > > > > to sum up a) and b) - doable but will require a lot of effort. > > > > > > > > Do you have any specific practically relevant example in mind that we > > > > should > > > > handle? BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

Re: [Maria-developers] [Commits] 53d0e921794: MDEV-11094: Blackhole table updates on slave fail when row annotation is enabled
by andrei.elkin＠pp.inet.fi 27 May '19

27 May '19

Sujatha, howdy. Thank you for spelling it out for me! > Hello Andrei, ... >>> >>> Analysis: >>> ========= >>> Enabling "replicate_annotate_row_events" on slave, Tells the slave to write >>> annotate rows events received from the master to its own binary log. The >>> received annotate events are applied after the Gtid event as shown below. >>> thd->query() will be set to the actual query received from the master, through >>> annotate event. Annotate_rows event should not be deleted after the event is >>> applied as the thd->query will be used to generate new Annotate_rows event >>> during applying the subsequent Rows events. >> [ here ] >> The Table_map and Rows_log_event that follow do not touch thd->query(). > > The Table_map and Rows_log_event that follow "Annotate_rows" event do > make use of "thd->query()". Frankly I did not expect that. Perhaps it would have not been a bad idea to make Annotate_rows_log_event::do_apply_event() logging of itself, which still makes sense even if the slave applier will filter out its group's Rows_log_event:s so in this case the slave side event group would consists of mere Annotate. The above is a mere remark though, I have to accept what we have. >> >> I also suggest to check out mysqlbinlog run. There seems to be no test >> for that though, so one needs to be written. The goal would be to prove >> a created binlog output is as expected (e.g to include Annotate). > > > Sure, I will add a test case. Super! The patch is approved then. Cheers, Andrei

1 0

Re: [Maria-developers] [Commits] 42129dae63c: MDEV-18741: Optimizer trace: multi-part key ranges are printed incorrectly
by Sergey Petrunia 27 May '19

27 May '19

Hi Varun, One cosmetic comment below. Ok to push after it is addressed. On Fri, May 24, 2019 at 03:51:00PM +0530, Varun wrote: > revision-id: 42129dae63c4331b01f9a504c9f3ef1c0c1f857a (mariadb-10.4.4-126-g42129dae63c) > parent(s): 592fe954ef82be1bc08b29a8e54f7729eb1e1343 > author: Varun Gupta > committer: Varun Gupta > timestamp: 2019-05-24 14:43:44 +0530 > message: > > MDEV-18741: Optimizer trace: multi-part key ranges are printed incorrectly > > Changed the function append_range_all_keyparts to use sel_arg_range_seq_init / sel_arg_range_seq_next to produce ranges. > Also adjusted to print format for the ranges, now the ranges are printed as: > (keypart1_min, keypart2_min,..) OP (keypart1_name,keypart2_name, ..) OP (keypart1_max,keypart2_max, ..) > > Also added more tests for range and index merge access for optimizer trace ... > diff --git a/sql/opt_range_mrr.cc b/sql/opt_range_mrr.cc > index 3a25da3edb2..5644e456dd4 100644 > --- a/sql/opt_range_mrr.cc > +++ b/sql/opt_range_mrr.cc > @@ -53,6 +53,8 @@ typedef struct st_sel_arg_range_seq > int i; /* Index of last used element in the above array */ > > bool at_start; /* TRUE <=> The traversal has just started */ > + /* TRUE if last checked tree->key can be used for ROR-scan */ > + bool is_ror_scan; > } SEL_ARG_RANGE_SEQ; > > This comment made sense at the previous location, but here it doesn't. Here the meaning is something like "Iteration functions will set this to FALSE" if ranges being traversed do not allow to construct a ROR-scan". BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

[Maria-developers] GSoC Introduction
by Rucha Deodhar 27 May '19

27 May '19

Hello, I am Rucha Deodhar. I am a GSoC student and will implement INSERT...RETURNING. In the first week I plan to extend the parser. Approximately the next 2 months, I plan to make changes the in the codes in insert.cc and other files related to the project, and remaining time for testing. As suggested, I am allotting 40% of the time for testing.This is my approximate plan, Regards, Rucha.

2 1

Re: [Maria-developers] Regarding approach used for Early NULL filtering in the range optimizer(MDEV-15777)
by Sergey Petrunia 24 May '19

24 May '19

On Fri, May 24, 2019 at 10:08:04AM -0700, Igor Babaev wrote: > On 05/24/2019 04:03 AM, Varun Gupta wrote: > > Hi Igor, > > After discussing with Sergey, we came up with these conclusions as to > > why we used the approach of going through all the keyuses in the KEYUSE > > array > > > > Cases to consider: > > we have an index on column a > > > > 1) a OP const > > where OP can be (</<=/=/between/ etc on which we do range analysis) . > > This case is already handled by the range optimizer. We do not create a > > NULL rejecting predicate for such conditions. > > > > 2) eq_join conditions (tbl1.a = tbl2.col2) > > This is the specific case we have tried to implement in MDEV-15777, we > > create NULL rejecting predicates for the keyuses for each table and then > > feed these predicates to range optimizer > There is no logical explanation for checking null rejection of fields used > in equalities with the help of array keyuse. The logic here is as follows: We already collect the set of KEYUSE::null_rejecting attributes. Its meaning is very close to what the patch needs. Because of this, the part of the patch for MDEV-15777 which analyzes the KEYUSE array is very small: 122 lines including the comment (I counted make_null_rejecting_conds() and add_cond()). I think, the CPU overhead is small, too. > > > > 3) a < func(b,c) > > we do not handle this case, because: > > > > a) It is harder to process. We would need to process the whole WHERE > > clause to infer > > non-nullability of the condition. Example: > > > > (t1.f2+1 < t1.f1+t1.f3) OR ... OR t1.f1 is null > > > > here the left part of the OR condition doesn't allow f1 to be NULL, but > > the > > right condition allows it. We would also need to take into account tables > > inside outer joins. We would need to go through Item classes and add code > > which say "Item_xxx() will compute to false when its argument Y is null" > > (there is item->not_null_tables() currently, but not not_null_columns()). > Have you actually looked at the implementations of the virtual function > not_null_tables()? > Do you need me to provide the implementation of the virtual function > not_null_columns()? It will require multiple implementations. At the moment we have: nm mysqld --demangle | grep 'Item.*::not_null_tables' | wc -l 21 Another question: do you intend to collect not_null_columns() for all columns, or just columns that are a part of some index? What would a good data structure to store a set of tablename.column_name ? > > > > b) the conditions in form of arbitrary functions are not as frequently > > used as > > ref access conditions. > > > > to sum up a) and b) - doable but will require a lot of effort. > > > > Do you have any specific practically relevant example in mind that we > > should > > handle? BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

[Maria-developers] Regarding approach used for Early NULL filtering in the range optimizer(MDEV-15777)
by Varun Gupta 24 May '19

24 May '19

Hi Igor, After discussing with Sergey, we came up with these conclusions as to why we used the approach of going through all the keyuses in the KEYUSE array Cases to consider: we have an index on column a 1) a OP const where OP can be (</<=/=/between/ etc on which we do range analysis) . This case is already handled by the range optimizer. We do not create a NULL rejecting predicate for such conditions. 2) eq_join conditions (tbl1.a = tbl2.col2) This is the specific case we have tried to implement in MDEV-15777, we create NULL rejecting predicates for the keyuses for each table and then feed these predicates to range optimizer 3) a < func(b,c) we do not handle this case, because: a) It is harder to process. We would need to process the whole WHERE clause to infer non-nullability of the condition. Example: (t1.f2+1 < t1.f1+t1.f3) OR ... OR t1.f1 is null here the left part of the OR condition doesn't allow f1 to be NULL, but the right condition allows it. We would also need to take into account tables inside outer joins. We would need to go through Item classes and add code which say "Item_xxx() will compute to false when its argument Y is null" (there is item->not_null_tables() currently, but not not_null_columns()). b) the conditions in form of arbitrary functions are not as frequently used as ref access conditions. to sum up a) and b) - doable but will require a lot of effort. Do you have any specific practically relevant example in mind that we should handle? Regards, Varun and Sergey

1 0

[Maria-developers] Review for: b4696ee97e684 : MDEV-15777: Support Early NULLs filtering-like restrictions in the range optimizer
by Sergey Petrunia 23 May '19

23 May '19

Hi Varun, Please find review input below. Only cosmetic changes are requested. I also assume that after this is pushed - we add a optimizer trace printout into 10.4 - the feature is enabled by default in 10.5 ? (do we need separate MDEVs for this?) One final question is the choice of name for @@optimizer_switch flag. Let me get back to you on this. > commit b4696ee97e6846ba49ef203cfc7189f50c1e53a7 > Author: Varun Gupta <varun.gupta(a)mariadb.com> > Date: Mon May 20 15:14:30 2019 +0530 > > MDEV-15777: Support Early NULLs filtering-like restrictions in the range optimizer > > For eqjoin conditions, we add a NOT NULL predicate so as to allow the range optimizer > to use the predicate and possibly create a range access on the given table. > > Example: > select * from t1,t2 where t1.a=t2.a; we have KEY(a) on t1 > we would inject a NOT NULL predicate t1.a IS NOT NULL for table t1 > this would allow the range optimizer to create ranges and we get a > range access on table t1, then we will be able > to do an early NULL filtering for ranges too. > > diff --git a/mysql-test/main/range.result b/mysql-test/main/range.result > index 32e0cf2868c..d75d6734277 100644 > --- a/mysql-test/main/range.result > +++ b/mysql-test/main/range.result > @@ -3024,3 +3024,151 @@ drop table t1; > # > # End of 10.2 tests > # > +# > +# MDEV-15777: Support Early NULLs filtering-like restrictions in the range optimizer > +# > +set @save_optimizer_switch= @@optimizer_switch; > +set @@optimizer_switch='null_rejecting_for_ranges=on'; > +create table ten(a int); > +insert into ten values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); > +create table one_k(a int); > +insert into one_k select A.a + B.a* 10 + C.a * 100 from ten A, ten B, ten C; Did we abandon the convention of naming the tables t1,t2,t3,... ? If not, please rename. > +create table t1 ( > +id int(10) unsigned NOT NULL, > +subset_id int(11) DEFAULT NULL, > +PRIMARY KEY (id), > +KEY t1_subset_id (subset_id)); ... > diff --git a/sql/opt_range.cc b/sql/opt_range.cc > index ec7b3dbbd7a..cb7800d1ba6 100644 > --- a/sql/opt_range.cc > +++ b/sql/opt_range.cc > @@ -2539,6 +2546,42 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, > TRP_GROUP_MIN_MAX *group_trp; > double best_read_time= read_time; > > + /* > + For the range optimizer, null_rejecting_conds are just an extra condition > + on which one can use index to create ranges. > + > + Also here we try to allow consider range access for a column involed in a > + null rejecting predicate only if it is the first keypart of an index. > + Let try to make it clear with examples: > + > + Query 1 > + select * from t1,t2 where t1.a=2 and t1.b=t2.b; > + and we have a key(a,b) > + So lets consider range access for table t1: > + Null rejecting predicate : t1.b IS NOT NULL > + but the column t1.b is NOT the first keypart in the index, > + so we don't consider it while building ranges > + > + Query 2 > + select * from t1,t2 where t1.a=t2.a and t1.b=t2.b; > + and we have a key (a,b) > + > + So lets consider range access for table t1: > + Null rejecting predicate : t1.a IS NOT NULL AND t1.b IS NOT NULL > + so here t1.a is the first keypart in the index, so we consider > + the predicate for building the ranges. > + > + The first keypart prerequisite is made as we try to reuse range > + estimates (because they are more accurate) at different places > + and we needed to make sure that correct/accurate estimates are used there. > + */ > + > + if (null_rejecting_conds) > + not_null_cond_tree= null_rejecting_conds->get_mm_tree(&param, > + &null_rejecting_conds); Please fix indentation and fit within 80 chars. ^ And also use curly braces as the code inside if doesn't fit on one line. > + if (not_null_cond_tree) > + remove_nonrange_trees(&param, not_null_cond_tree); > + > if (cond) > { > if ((tree= cond->get_mm_tree(&param, &cond))) > @@ -2557,6 +2600,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, > tree= NULL; > } > } > + tree= tree_and(&param, tree, not_null_cond_tree); > > /* > Try to construct a QUICK_GROUP_MIN_MAX_SELECT. > @@ -4422,6 +4466,128 @@ static bool create_partition_index_description(PART_PRUNE_PARAM *ppar) > return FALSE; > } > > +inline void add_cond(THD *thd, Item **e1, Item *e2) > +{ > + if (*e1) > + { > + if (!e2) > + return; > + Item *res; > + if ((res= new (thd->mem_root) Item_cond_and(thd, *e1, e2))) > + { > + res->fix_fields(thd, 0); > + res->update_used_tables(); > + *e1= res; > + } > + } > + else > + *e1= e2; > +} > + > +/* > + Create null rejecting conditions for a table, for all the equalites > + present in the WHERE clause of a query. > + > + SYNOPSIS > + make_null_rejecting_conds() > + @param TABLE - Keys of this table will participate in null > + rejecting conditions > + @param keyuse_array - array that has all the equalites of the > + WHERE clasuse The above doesn't match the function's definition. Out of date comment? > + > + DESCRIPTION > + This function creates null rejecting conditions for a table. These > + conditions are created to do range analysis on them , the conditions > + are of the form tbl.key.keypart IS NOT NULL. > + > + IMPLEMENTATION > + Lookup in the keyuse array to check if it has equalites that belong > + to the given table. If yes then find out if the conditions are null > + rejecting and accordingly create all the condition for the keys of a > + given table and AND them. > + > + > + RETURN > + NOT NULL - Found null rejecting conditions for the given table > + NULL - No null rejecting conditions for the given table The above doesn't match the function's definition. Out of date comment? > +*/ > + > +void make_null_rejecting_conds(THD *thd, JOIN_TAB *tab) > +{ > + KEY *keyinfo; > + Item *cond= NULL; > + KEYUSE* keyuse; > + TABLE *table= tab->table; > + key_map *const_keys= &tab->const_keys; ^ Does it make sense to introduce a variable if it is only used once? > + > + /* > + No need to add NOT NULL condition for materialized derived tables > + or materialized subqueries as we do not run the range optimizer > + on their conditions > + */ > + > + if (!optimizer_flag(thd, OPTIMIZER_SWITCH_NULL_REJECTING_FOR_RANGES)) > + return; > + > + if (tab->table->is_filled_at_execution() || > + (tab->table->pos_in_table_list->derived && > + tab->table->pos_in_table_list->is_materialized_derived())) > + return; > + > + Field_map not_null_keypart_map; > + > + /* > + The null rejecting conds added will be on the keypart of a key, so for > + that we need the table to atleast have a key. > + */ > + if (!table->s->keys || table->null_rejecting_conds || !tab->keyuse) > + return; > + > + for(keyuse= tab->keyuse; keyuse->table == table; keyuse++) > + { Please add space, "for ". > + /* > + No null rejecting conds for a hash key or full-text keys > + */ > + if (keyuse->key == MAX_KEY || keyuse->keypart == FT_KEYPART) > + continue; > + keyinfo= keyuse->table->key_info + keyuse->key; > + Field *field= keyinfo->key_part[keyuse->keypart].field; > + > + if (not_null_keypart_map.is_set(field->field_index)) > + continue; > + > + /* > + No need to add null-rejecting condition if we have a > + keyuse element as > + - table.key.keypart= const > + - (table.key.keypart= tbl.otherfield or table.key.keypart IS NULL) > + - table.key.keypart IS NOT NULLABLE > + */ > + > + if (keyuse->val->const_item() || > + !(keyuse->null_rejecting && field->maybe_null()) || > + keyuse->optimize & KEY_OPTIMIZE_REF_OR_NULL) > + continue; > + > + Item_field *field_item= new (thd->mem_root)Item_field(thd, field); > + Item* not_null_item= new (thd->mem_root)Item_func_isnotnull(thd, field_item); > + > + /* > + adding the key to const keys as we have the condition > + as key.keypart IS NOT NULL > + */ > + > + const_keys->set_bit(keyuse->key); > + not_null_item->fix_fields(thd, 0); ^^ please change 0 to &not_null_item. > + not_null_item->update_used_tables(); > + add_cond(thd, &cond, not_null_item); > + not_null_keypart_map.set_bit(field->field_index); > + } > + DBUG_EXECUTE("where", print_where(cond, "Early Null Filtering", > + QT_ORDINARY);); > + table->null_rejecting_conds= cond; > +} > + > > #ifndef DBUG_OFF > ... > diff --git a/sql/sql_select.cc b/sql/sql_select.cc > index 8cdcf3afc8b..140ceed5c2d 100644 > --- a/sql/sql_select.cc > +++ b/sql/sql_select.cc > @@ -5510,15 +5512,30 @@ add_key_field(JOIN *join, > If the condition has form "tbl.keypart = othertbl.field" and > othertbl.field can be NULL, there will be no matches if othertbl.field > has NULL value. > - We use null_rejecting in add_not_null_conds() to add > - 'othertbl.field IS NOT NULL' to tab->select_cond. > + > + The field KEY_FIELD::null_rejecting is set to TRUE if either > + the left or the right hand side of the equality is NULLABLE > + > + null_rejecting is used > + - by add_not_null_conds(), to add "othertbl.field IS NOT NULL" to > + othertbl's tab->select_cond. (This is called "Early NULLs filtering") > + > + - by make_null_rejecting_conds(), to provide range optimizer with > + additional "tbl.keypart IS NOT NULL" condition. > */ > { > Item *real= (*value)->real_item(); > - if (((cond->functype() == Item_func::EQ_FUNC) || > - (cond->functype() == Item_func::MULT_EQUAL_FUNC)) && > - (real->type() == Item::FIELD_ITEM) && > - ((Item_field*)real)->field->maybe_null()) > + if (( > + (cond->functype() == Item_func::EQ_FUNC) || > + (cond->functype() == Item_func::MULT_EQUAL_FUNC) > + ) && > + ( > + ( > + (real->type() == Item::FIELD_ITEM) && > + ((Item_field*)real)->field->maybe_null() > + ) || > + (field->maybe_null()) > + )) > (*key_fields)->null_rejecting= true; > else > (*key_fields)->null_rejecting= false; > @@ -9982,8 +9999,12 @@ static bool create_ref_for_key(JOIN *join, JOIN_TAB *j, > uint maybe_null= MY_TEST(keyinfo->key_part[i].null_bit); > j->ref.items[i]=keyuse->val; // Save for cond removal > j->ref.cond_guards[i]= keyuse->cond_guard; > - if (keyuse->null_rejecting) > + Item *real= (keyuse->val)->real_item(); > + if (keyuse->null_rejecting && > + (real->type() == Item::FIELD_ITEM) && > + ((Item_field*)real)->field->maybe_null()) This code looks as if if keyuse->val->maybe_null() != keyuse->val->real_item()->maybe_null() do you have any specific case in mind or this is just to match the code in add_key_field? > j->ref.null_rejecting|= (key_part_map)1 << i; > + > keyuse_uses_no_tables= keyuse_uses_no_tables && !keyuse->used_tables; > /* > We don't want to compute heavy expressions in EXPLAIN, an example would BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

2 1

Re: [Maria-developers] [Commits] 53d0e921794: MDEV-11094: Blackhole table updates on slave fail when row annotation is enabled
by andrei.elkin＠pp.inet.fi 22 May '19

22 May '19

Sujatha, salute! Thanks for a detailed description! I would add up to it a minor piece > revision-id: 53d0e9217943719b806ef9fa1cac0a699df4839c (mariadb-10.1.39-31-g53d0e921794) > parent(s): 47637a3dd13d19e897a7cbfd1499f1bf3b2fdb2a > author: Sujatha > committer: Sujatha > timestamp: 2019-05-17 13:11:49 +0530 > message: > > MDEV-11094: Blackhole table updates on slave fail when row annotation is enabled > > Problem: > ======= > rpl_blackhole.test fails when executed with following options > mysqld=--binlog_annotate_row_events=1, mysqld=--replicate_annotate_row_events=1 > > Test output: > ------------ > worker[1] Using MTR_BUILD_THREAD 300, with reserved ports 16000..16019 > rpl.rpl_blackhole_bug 'mix' [ pass ] 791 > rpl.rpl_blackhole_bug 'row' [ fail ] > Replicate_Wild_Ignore_Table > Last_Errno 1032 > Last_Error Could not execute Update_rows_v1 event on table test.t1; Can't find > record in 't1', Error_code: 1032; handler error HA_ERR_END_OF_FILE; the event's > master log master-bin.000001, end_log_pos 1510 > > Analysis: > ========= > Enabling "replicate_annotate_row_events" on slave, Tells the slave to write > annotate rows events received from the master to its own binary log. The > received annotate events are applied after the Gtid event as shown below. > thd->query() will be set to the actual query received from the master, through > annotate event. Annotate_rows event should not be deleted after the event is > applied as the thd->query will be used to generate new Annotate_rows event > during applying the subsequent Rows events. [ here ] The Table_map and Rows_log_event that follow do not touch thd->query(). Notice that the being suggested sentence is sufficient to explain the not-NULL without the sentence below (fill free to remove then): > After the last Rows event has been > applied, the saved Annotate_rows event (if any) will be deleted. > > master-bin.000001 | 457 | Gtid | BEGIN GTID 0-1-2 > master-bin.000001 | 495 | Annotate_rows | insert into t values (10) > master-bin.000001 | 539 | Table_map | table_id: 19 (test.t) > master-bin.000001 | 579 | Write_rows_v1 | table_id: 19 flags: STMT_END_F > master-bin.000001 | 613 | Xid | COMMIT /* xid=7 */ > > In balckhole engine all the DML operations are noops as they donot store any > data. They simply return success without doing any operation. But the existing > strictly expects thd->query() to be 'NULL' to identify that row based > replication is in use. This assumption will fail when row annotations are > enabled as the query is not 'NULL'. Hence various row based operations like > 'update', 'delete', 'index lookup' will fail when row annotations are enabled. > > Fix: > === > Extend the row based replication check to include row annotations as well. > i.e Either the thd->query() is NULL or thd->query() points to query and row > annotations are in use. This works, but the check does not really refers to the Annotate. So in principle thd->query() may turn in not-NULL of different reasons. Besides the check is multiplied over few places (again hopefully covering all). We can do it simpler actually: Rows_log_event::do_apply_event() { ... do { /* in_use can have been set to NULL in close_tables_for_reopen */ THD* old_thd= table->in_use; if (!table->in_use) table->in_use= thd; --> add a check whether Annotate has been done and when so store/restore --> thd->query() like `old_thd` above/below, and pass it as NULL (thd->reset_query()) to the following: error= do_exec_row(rgi); if (error) DBUG_PRINT("info", ("error: %s", HA_ERR(error))); DBUG_ASSERT(error != HA_ERR_RECORD_DELETED); table->in_use = old_thd; I also suggest to check out mysqlbinlog run. There seems to be no test for that though, so one needs to be written. The goal would be to prove a created binlog output is as expected (e.g to include Annotate). Cheers, Andrei > > --- > mysql-test/suite/rpl/t/rpl_blackhole-master.opt | 1 + > mysql-test/suite/rpl/t/rpl_blackhole-slave.opt | 1 + > storage/blackhole/ha_blackhole.cc | 28 +++++++++++++++++++------ > 3 files changed, 24 insertions(+), 6 deletions(-) > > diff --git a/mysql-test/suite/rpl/t/rpl_blackhole-master.opt b/mysql-test/suite/rpl/t/rpl_blackhole-master.opt > new file mode 100644 > index 00000000000..1ad0b884c60 > --- /dev/null > +++ b/mysql-test/suite/rpl/t/rpl_blackhole-master.opt > @@ -0,0 +1 @@ > +--binlog_annotate_row_events > diff --git a/mysql-test/suite/rpl/t/rpl_blackhole-slave.opt b/mysql-test/suite/rpl/t/rpl_blackhole-slave.opt > new file mode 100644 > index 00000000000..7ac6a84faa7 > --- /dev/null > +++ b/mysql-test/suite/rpl/t/rpl_blackhole-slave.opt > @@ -0,0 +1 @@ > +--binlog_annotate_row_events --replicate_annotate_row_events > diff --git a/storage/blackhole/ha_blackhole.cc b/storage/blackhole/ha_blackhole.cc > index 01aaa9ea15f..43bcdc541a1 100644 > --- a/storage/blackhole/ha_blackhole.cc > +++ b/storage/blackhole/ha_blackhole.cc > @@ -25,6 +25,16 @@ > #include "ha_blackhole.h" > #include "sql_class.h" // THD, SYSTEM_THREAD_SLAVE_SQL > > +static bool is_row_based_replication(THD *thd) > +{ > + /* > + A row event which has its thd->query() == NULL or a row event which has > + replicate_annotate_row_events enabled. In the later case the thd->query() > + will be pointing to the query, received through replicated annotate event > + from master. > + */ > + return ((thd->query() == NULL) || thd->variables.binlog_annotate_row_events); > +} > /* Static declarations for handlerton */ > > static handler *blackhole_create_handler(handlerton *hton, > @@ -109,7 +119,8 @@ int ha_blackhole::update_row(const uchar *old_data, uchar *new_data) > { > DBUG_ENTER("ha_blackhole::update_row"); > THD *thd= ha_thd(); > - if (thd->system_thread == SYSTEM_THREAD_SLAVE_SQL && thd->query() == NULL) > + if (thd->system_thread == SYSTEM_THREAD_SLAVE_SQL && > + is_row_based_replication(thd)) > DBUG_RETURN(0); > DBUG_RETURN(HA_ERR_WRONG_COMMAND); > } > @@ -118,7 +129,8 @@ int ha_blackhole::delete_row(const uchar *buf) > { > DBUG_ENTER("ha_blackhole::delete_row"); > THD *thd= ha_thd(); > - if (thd->system_thread == SYSTEM_THREAD_SLAVE_SQL && thd->query() == NULL) > + if (thd->system_thread == SYSTEM_THREAD_SLAVE_SQL && > + is_row_based_replication(thd)) > DBUG_RETURN(0); > DBUG_RETURN(HA_ERR_WRONG_COMMAND); > } > @@ -135,7 +147,8 @@ int ha_blackhole::rnd_next(uchar *buf) > int rc; > DBUG_ENTER("ha_blackhole::rnd_next"); > THD *thd= ha_thd(); > - if (thd->system_thread == SYSTEM_THREAD_SLAVE_SQL && thd->query() == NULL) > + if (thd->system_thread == SYSTEM_THREAD_SLAVE_SQL && > + is_row_based_replication(thd)) > rc= 0; > else > rc= HA_ERR_END_OF_FILE; > @@ -220,7 +233,8 @@ int ha_blackhole::index_read_map(uchar * buf, const uchar * key, > int rc; > DBUG_ENTER("ha_blackhole::index_read"); > THD *thd= ha_thd(); > - if (thd->system_thread == SYSTEM_THREAD_SLAVE_SQL && thd->query() == NULL) > + if (thd->system_thread == SYSTEM_THREAD_SLAVE_SQL && > + is_row_based_replication(thd)) > rc= 0; > else > rc= HA_ERR_END_OF_FILE; > @@ -235,7 +249,8 @@ int ha_blackhole::index_read_idx_map(uchar * buf, uint idx, const uchar * key, > int rc; > DBUG_ENTER("ha_blackhole::index_read_idx"); > THD *thd= ha_thd(); > - if (thd->system_thread == SYSTEM_THREAD_SLAVE_SQL && thd->query() == NULL) > + if (thd->system_thread == SYSTEM_THREAD_SLAVE_SQL && > + is_row_based_replication(thd)) > rc= 0; > else > rc= HA_ERR_END_OF_FILE; > @@ -249,7 +264,8 @@ int ha_blackhole::index_read_last_map(uchar * buf, const uchar * key, > int rc; > DBUG_ENTER("ha_blackhole::index_read_last"); > THD *thd= ha_thd(); > - if (thd->system_thread == SYSTEM_THREAD_SLAVE_SQL && thd->query() == NULL) > + if (thd->system_thread == SYSTEM_THREAD_SLAVE_SQL && > + is_row_based_replication(thd)) > rc= 0; > else > rc= HA_ERR_END_OF_FILE; > _______________________________________________ > commits mailing list > commits(a)mariadb.org > https://lists.askmonty.org/cgi-bin/mailman/listinfo/commits

1 0

Re: [Maria-developers] 9e176e81b72: MDEV-18531 : Use WolfSSL instead of YaSSL as "bundled" SSL/encryption library
by Sergei Golubchik 21 May '19

21 May '19

Hi, Vladislav! It's good to see how small the patch is, and that wolfssl is more OpenSSL compatible than YaSSL. Few comments below. On May 19, Vladislav Vaintroub wrote: > revision-id: 9e176e81b72 (mariadb-10.4.4-61-g9e176e81b72) > parent(s): ea679c88c32 > author: Vladislav Vaintroub <wlad(a)mariadb.com> > committer: Vladislav Vaintroub <wlad(a)mariadb.com> > timestamp: 2019-05-01 11:28:56 +0100 > message: > > MDEV-18531 : Use WolfSSL instead of YaSSL as "bundled" SSL/encryption library > > - Add new submodule for WolfSSL > - Build and use wolfssl and wolfcrypt instead of yassl/taocrypt > - Use HAVE_WOLFSSL instead of HAVE_YASSL > - Increase MY_AES_CTX_SIZE, to avoid compile time asserts in my_crypt.cc > (sizeof(EVP_CIPHER_CTX) is larger on WolfSSL) > > diff --git a/cmake/ssl.cmake b/cmake/ssl.cmake > index aa42333a5c8..79a531ec323 100644 > --- a/cmake/ssl.cmake > +++ b/cmake/ssl.cmake > @@ -48,29 +48,19 @@ MACRO (CHANGE_SSL_SETTINGS string) > ENDMACRO() > > MACRO (MYSQL_USE_BUNDLED_SSL) > - SET(INC_DIRS > - ${CMAKE_SOURCE_DIR}/extra/yassl/include > - ${CMAKE_SOURCE_DIR}/extra/yassl/taocrypt/include > + SET(INC_DIRS > + ${CMAKE_SOURCE_DIR}/extra/wolfssl/wolfssl > + ${CMAKE_SOURCE_DIR}/extra/wolfssl/wolfssl/wolfssl > + #${CMAKE_SOURCE_DIR}/extra/wolfssl/taocrypt/include don't you want to remove these (and many below) commented lines? > ) > - SET(SSL_LIBRARIES yassl taocrypt) > + SET(SSL_LIBRARIES wolfssl wolfcrypt) > SET(SSL_INCLUDE_DIRS ${INC_DIRS}) > - SET(SSL_INTERNAL_INCLUDE_DIRS ${CMAKE_SOURCE_DIR}/extra/yassl/taocrypt/mySTL) > + SET(SSL_DEFINES "-DHAVE_OPENSSL -DHAVE_WOLFSSL -DOPENSSL_ALL -DWOLFSSL_MYSQL_COMPATIBLE -DWC_NO_HARDEN") > - SET(SSL_DEFINES "-DHAVE_YASSL -DYASSL_PREFIX -DHAVE_OPENSSL -DMULTI_THREADED") > - SET(HAVE_ERR_remove_thread_state OFF CACHE INTERNAL "yassl doesn't have ERR_remove_thread_state") > - SET(HAVE_EncryptAes128Ctr OFF CACHE INTERNAL "yassl doesn't support AES-CTR") > - SET(HAVE_EncryptAes128Gcm OFF CACHE INTERNAL "yassl doesn't support AES-GCM") > + SET(HAVE_ERR_remove_thread_state ON CACHE INTERNAL "wolfssl doesn't have ERR_remove_thread_state") > + SET(HAVE_EncryptAes128Ctr ON CACHE INTERNAL "wolfssl does support AES-CTR") > + SET(HAVE_EncryptAes128Gcm OFF CACHE INTERNAL "wolfssl does not support AES-GCM") > CHANGE_SSL_SETTINGS("bundled") > - ADD_SUBDIRECTORY(extra/yassl) > - ADD_SUBDIRECTORY(extra/yassl/taocrypt) > - GET_TARGET_PROPERTY(src yassl SOURCES) > - FOREACH(file ${src}) > - SET(SSL_SOURCES ${SSL_SOURCES} ${CMAKE_SOURCE_DIR}/extra/yassl/${file}) > - ENDFOREACH() > - GET_TARGET_PROPERTY(src taocrypt SOURCES) > - FOREACH(file ${src}) > - SET(SSL_SOURCES ${SSL_SOURCES} > - ${CMAKE_SOURCE_DIR}/extra/yassl/taocrypt/${file}) > - ENDFOREACH() > + ADD_SUBDIRECTORY(extra/wolfssl) > MESSAGE_ONCE(SSL_LIBRARIES "SSL_LIBRARIES = ${SSL_LIBRARIES}") > ENDMACRO() > > diff --git a/extra/wolfssl/CMakeLists.txt b/extra/wolfssl/CMakeLists.txt > new file mode 100644 > index 00000000000..c1165ad6c72 > --- /dev/null > +++ b/extra/wolfssl/CMakeLists.txt > @@ -0,0 +1,119 @@ > +# CMakeLists.txt > +# > +# Copyright (C) 2006-2015 wolfSSL Inc. > +# > +# This file is part of wolfSSL. (formerly known as CyaSSL) > +# > +# wolfSSL is free software; you can redistribute it and/or modify > +# it under the terms of the GNU General Public License as published by > +# the Free Software Foundation; either version 2 of the License, or > +# (at your option) any later version. > +# > +# wolfSSL is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +# GNU General Public License for more details. > +# > +# You should have received a copy of the GNU General Public License > +# along with this program; if not, write to the Free Software > +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA > + > +SET(WOLFSSL_SRCDIR ${CMAKE_CURRENT_SOURCE_DIR}/wolfssl/src) > +ADD_DEFINITIONS(${SSL_DEFINES}) > +ADD_DEFINITIONS( > + -DWOLFSSL_MYSQL_COMPATIBLE > + -DHAVE_ECC > + -DECC_TIMING_RESISTANT > + -DBUILDING_WOLFSSL > + -DHAVE_HASHDRBG > + -DWOLFSSL_AES_DIRECT > + -DWOLFSSL_SHA384 > + -DWOLFSSL_SHA512 > + -DWOLFSSL_SHA224 > + -DSESSION_CERT > + -DKEEP_OUR_CERT > + -DWOLFSSL_STATIC_RSA > + -DWC_RSA_BLINDING > + -DHAVE_TLS_EXTENSIONS > + -DHAVE_AES_ECB > + -DWOLFSSL_AES_COUNTER > + -DNO_WOLFSSL_STUB) > + > +SET(WOLFSSL_SOURCES > + ${WOLFSSL_SRCDIR}/crl.c > + ${WOLFSSL_SRCDIR}/internal.c > + ${WOLFSSL_SRCDIR}/keys.c > + ${WOLFSSL_SRCDIR}/tls.c > + ${WOLFSSL_SRCDIR}/wolfio.c > + ${WOLFSSL_SRCDIR}/ocsp.c > + ${WOLFSSL_SRCDIR}/ssl.c) > +ADD_DEFINITIONS(-DWOLFSSL_LIB) > +INCLUDE_DIRECTORIES(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/wolfssl) > +IF(MSVC) > + # size_t to long truncation warning > + ADD_DEFINITIONS(-wd4267) > +ENDIF() > + > + > +ADD_CONVENIENCE_LIBRARY(wolfssl ${WOLFSSL_SOURCES}) > + > +# Workaround linker crash with older Ubuntu binutils > +# e.g aborting at ../../bfd/merge.c line 873 in _bfd_merged_section_offset > +IF((CMAKE_SYSTEM_NAME MATCHES "Linux") AND (CMAKE_LINKER STREQUAL "/usr/bin/ld")) > + EXECUTE_PROCESS(COMMAND ${CMAKE_LINKER} -v OUTPUT_VARIABLE LINKER_VERSION_STR) > + STRING(REGEX MATCH "[0-9]+\\.[0-9]+" LINKER_VERSION > + "${LINKER_VERSION_STR}") > + IF(LINKER_VERSION AND (LINKER_VERSION VERSION_LESS 2.25)) > + STRING(REPLACE "-g " "-g1 " CMAKE_C_FLAGS_RELWITHDEBINFO > + ${CMAKE_C_FLAGS_RELWITHDEBINFO}) > + STRING(REPLACE "-g " "-g1 " CMAKE_C_FLAGS_DEBUG > + ${CMAKE_C_FLAGS_DEBUG}) > + STRING(REPLACE "-ggdb3 " " " CMAKE_C_FLAGS_RELWITHDEBINFO > + ${CMAKE_C_FLAGS_RELWITHDEBINFO}) > + STRING(REPLACE "-ggdb3 " " " CMAKE_C_FLAGS_DEBUG > + ${CMAKE_C_FLAGS_DEBUG}) > + ENDIF() > +ENDIF() I would've just disabled -g\w+ unconditionally, for a simpler CMakeLists.txt We aren't going to debug wolfssl anyway (and if I'd need to debug it, for some weird reason, I know how to enable debugging temporarily) > + > + > +SET(WOLFCRYPT_SRCDIR ${CMAKE_CURRENT_SOURCE_DIR}/wolfssl/wolfcrypt/src) > +SET(WOLFCRYPT_SOURCES > +${WOLFCRYPT_SRCDIR}/aes.c > +${WOLFCRYPT_SRCDIR}/arc4.c > +${WOLFCRYPT_SRCDIR}/asn.c > +#${WOLFCRYPT_SRCDIR}/blake2b.c > +#${WOLFCRYPT_SRCDIR}/camellia.c > +#${WOLFCRYPT_SRCDIR}/chacha.c > +${WOLFCRYPT_SRCDIR}/coding.c > +#${WOLFCRYPT_SRCDIR}/compress.c > +${WOLFCRYPT_SRCDIR}/des3.c > +${WOLFCRYPT_SRCDIR}/dh.c > +${WOLFCRYPT_SRCDIR}/dsa.c > +${WOLFCRYPT_SRCDIR}/ecc.c > +${WOLFCRYPT_SRCDIR}/error.c > +#${WOLFCRYPT_SRCDIR}/hc128.c > +${WOLFCRYPT_SRCDIR}/hmac.c > +${WOLFCRYPT_SRCDIR}/integer.c > +${WOLFCRYPT_SRCDIR}/logging.c > +#${WOLFCRYPT_SRCDIR}/md2.c > +${WOLFCRYPT_SRCDIR}/md4.c > +${WOLFCRYPT_SRCDIR}/md5.c > +${WOLFCRYPT_SRCDIR}/memory.c > +#${WOLFCRYPT_SRCDIR}/pkcs7.c > +${WOLFCRYPT_SRCDIR}/pkcs12.c > +#${WOLFCRYPT_SRCDIR}/poly1305.c > +${WOLFCRYPT_SRCDIR}/pwdbased.c > +${WOLFCRYPT_SRCDIR}/rabbit.c > +${WOLFCRYPT_SRCDIR}/random.c > +#${WOLFCRYPT_SRCDIR}/ripemd.c > +${WOLFCRYPT_SRCDIR}/rsa.c > +${WOLFCRYPT_SRCDIR}/sha.c > +${WOLFCRYPT_SRCDIR}/sha256.c > +${WOLFCRYPT_SRCDIR}/sha512.c > +#${WOLFCRYPT_SRCDIR}/tfm.c > +${WOLFCRYPT_SRCDIR}/wc_port.c > +${WOLFCRYPT_SRCDIR}/wc_encrypt.c > +${WOLFCRYPT_SRCDIR}/hash.c > +${WOLFCRYPT_SRCDIR}/wolfmath.c) > +ADD_CONVENIENCE_LIBRARY(wolfcrypt ${WOLFCRYPT_SOURCES}) > + > diff --git a/extra/wolfssl/wolfssl b/extra/wolfssl/wolfssl > new file mode 160000 > index 00000000000..dc313ccf6ee > --- /dev/null > +++ b/extra/wolfssl/wolfssl > @@ -0,0 +1 @@ > +Subproject commit dc313ccf6ee0e99fb5a13793b82e412f396e11c6 I think we should only use wolfssl (any external project, actually) only at release points, not at intermediate commits. 4.0.0-stable should be ok, I guess. And, generaly, pefer to use system globally installed library, if available. Although in case of wolfssl (and libmarias3) I would not bother. > diff --git a/include/ssl_compat.h b/include/ssl_compat.h > index c94b9671d5f..105405efff2 100644 > --- a/include/ssl_compat.h > +++ b/include/ssl_compat.h > @@ -17,9 +17,9 @@ > #include <openssl/opensslv.h> > > /* OpenSSL version specific definitions */ > -#if !defined(HAVE_YASSL) && defined(OPENSSL_VERSION_NUMBER) > +#if defined(OPENSSL_VERSION_NUMBER) > > -#if OPENSSL_VERSION_NUMBER >= 0x10002000L && !defined(LIBRESSL_VERSION_NUMBER) > +#if (OPENSSL_VERSION_NUMBER >= 0x10002000L && !defined(LIBRESSL_VERSION_NUMBER)) || defined (HAVE_WOLFSSL) What's OPENSSL_VERSION_NUMBER in wolfssl? > #define HAVE_X509_check_host 1 May be move it to ssl.cmake? Where other such defines are. > #endif > > diff --git a/include/sslopt-case.h b/include/sslopt-case.h > index 4a8c65948cb..b129a97cc76 100644 > --- a/include/sslopt-case.h > +++ b/include/sslopt-case.h > @@ -29,8 +29,8 @@ > One can disable SSL later by using --skip-ssl or --ssl=0 > */ > opt_use_ssl= 1; > - /* crl has no effect in yaSSL */ > -#ifdef HAVE_YASSL > +#ifdef HAVE_WOLFSSL > + /* CRL does not work with WolfSSL */ Still? > opt_ssl_crl= NULL; > opt_ssl_crlpath= NULL; > #endif > diff --git a/mysys_ssl/my_crypt.cc b/mysys_ssl/my_crypt.cc > index 2d6f5188034..1ce0c6bbc03 100644 > --- a/mysys_ssl/my_crypt.cc > +++ b/mysys_ssl/my_crypt.cc > @@ -18,14 +18,10 @@ > #include <my_global.h> > #include <string.h> > > -#ifdef HAVE_YASSL > -#include "yassl.cc" delete yassl.cc file, perhaps? > -#else > #include <openssl/evp.h> > #include <openssl/aes.h> > #include <openssl/err.h> > #include <openssl/rand.h> > -#endif > > #include <my_crypt.h> > #include <ssl_compat.h> > @@ -70,8 +66,24 @@ class MyCTX > } > virtual int finish(uchar *dst, uint *dlen) > { > +#ifdef HAVE_WOLFSSL > + /* > + Bug in WolfSSL - sometimes EVP_CipherFinal_ex > + returns success without setting destination length > + when it should return error. > + We catch it by presetting invalid value for length, > + and checking if it has changed after the call. > + > + See https://github.com/wolfSSL/wolfssl/issues/2092 > + */ > + *dlen= UINT_MAX; > +#endif I suppose you can remove it. This is fixed as of 4.0.0-stable tag. > if (!EVP_CipherFinal_ex(ctx, dst, (int*)dlen)) > return MY_AES_BAD_DATA; > +#ifdef HAVE_WOLFSSL > + if (*dlen == UINT_MAX) > + return MY_AES_BAD_DATA; > +#endif > return MY_AES_OK; > } > }; > diff --git a/plugin/file_key_management/parser.cc b/plugin/file_key_management/parser.cc > index 13a9dfa0cb6..27277f62994 100644 > --- a/plugin/file_key_management/parser.cc > +++ b/plugin/file_key_management/parser.cc > @@ -103,7 +103,6 @@ openssl enc -aes-256-cbc -md sha1 -k "secret" -in keys.txt -out keys.enc > EVP_BytesToKey(EVP_aes_256_cbc(), EVP_sha1(), salt, > secret, strlen(secret), 1, key, iv); > > - but alas! we want to support yassl too does wolfssl have an equivalent of EVP_BytesToKey? if not, the comment still holds. > */ > > void Parser::bytes_to_key(const unsigned char *salt, const char *input, > diff --git a/sql/mysqld.cc b/sql/mysqld.cc > index bfa03aa57c1..8c3a59c55d1 100644 > --- a/sql/mysqld.cc > +++ b/sql/mysqld.cc > @@ -7317,13 +7315,13 @@ static int show_ssl_get_verify_mode(THD *thd, SHOW_VAR *var, char *buff, > { > var->type= SHOW_LONG; > var->value= buff; > -#ifndef HAVE_YASSL > +#ifndef HAVE_WOLFSSL > if( thd->net.vio && thd->net.vio->ssl_arg ) > *((long *)buff)= (long)SSL_get_verify_mode((SSL*)thd->net.vio->ssl_arg); no SSL_get_verify_mode or an analog in wolfssl? I'm asking, as I see it has gone a long way towards OpenSSL compatibility > else > *((long *)buff)= 0; > #else > - *((long *)buff) = 0; > + *((long *)buff)= 0; > #endif > return 0; > } Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

2 2

Re: [Maria-developers] ddec45aa50e: MDEV-16932: ASAN heap-use-after-free in my_charlen_utf8 / my_well_formed_char_length_utf8 on 2nd execution of SP with ALTER trying to add bad CHECK
by Sergei Golubchik 20 May '19

20 May '19

Hi, Oleksandr! On May 20, Oleksandr Byelkin wrote: > Alter table will trigger recompilation of the SP (as I remember all SPs) I've run this test, it complains about duplicate constraint name. > > What about this: > > > > CREATE TABLE t1 (a INT); > > CREATE PROCEDURE sp() ALTER TABLE t1 ADD CONSTRAINT CHECK (b > 0); > > --error ER_BAD_FIELD_ERROR > > CALL sp; > > alter table t1 add column b int, add constraint check (b < 10); > > CALL sp; > > > > ? > > second CALL will not regenerate the name of the constraint. > > Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

1 0

Re: [Maria-developers] ddec45aa50e: MDEV-16932: ASAN heap-use-after-free in my_charlen_utf8 / my_well_formed_char_length_utf8 on 2nd execution of SP with ALTER trying to add bad CHECK
by Sergei Golubchik 20 May '19

20 May '19

Hi, Oleksandr! On May 20, Oleksandr Byelkin wrote: > On Mon, May 20, 2019 at 3:11 PM Sergei Golubchik <serg(a)mariadb.org> wrote: > > On May 20, Oleksandr Byelkin wrote: > > > revision-id: ddec45aa50e (mariadb-10.2.24-11-gddec45aa50e) > > > parent(s): 50999738eae > > > author: Oleksandr Byelkin <sanja(a)mariadb.com> > > > committer: Oleksandr Byelkin <sanja(a)mariadb.com> > > > timestamp: 2019-05-14 14:01:15 +0200 > > > message: > > > > > > MDEV-16932: ASAN heap-use-after-free in my_charlen_utf8 / > > > my_well_formed_char_length_utf8 on 2nd execution of SP with ALTER > > > trying to add bad CHECK > > > > > > In case of error the SP can be executed without re-comilation and so > > > will reuse constructed constaint name, so the name should be allocated > > > in the statement memory. > > > > Is that right? You're saying that the "unique constraint name" will be > > generated once and reused? > > > > But on the second SP invocation is might no longer be unique. I'd think > > that `ALTER TABLE t1 ADD CONSTRAINT CHECK (x > 0)` should add a new > > constraint, not "add a new contraint with the name that used to be > > unique when the SP was invoked for the first time". > > With the second call we have 2 possibility: > > 1) first execution failed and so we can reuse the name > 2) first execution succeed and we will re-compile SP or re-prepare PS and > so will generate new name (that is why we mostly had no problems) What about this: CREATE TABLE t1 (a INT); CREATE PROCEDURE sp() ALTER TABLE t1 ADD CONSTRAINT CHECK (b > 0); --error ER_BAD_FIELD_ERROR CALL sp; alter table t1 add column b int, add constraint check (b < 10); CALL sp; ? second CALL will not regenerate the name of the constraint. Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

1 0

Re: [Maria-developers] ddec45aa50e: MDEV-16932: ASAN heap-use-after-free in my_charlen_utf8 / my_well_formed_char_length_utf8 on 2nd execution of SP with ALTER trying to add bad CHECK
by Sergei Golubchik 20 May '19

20 May '19

Hi, Oleksandr! On May 20, Oleksandr Byelkin wrote: > revision-id: ddec45aa50e (mariadb-10.2.24-11-gddec45aa50e) > parent(s): 50999738eae > author: Oleksandr Byelkin <sanja(a)mariadb.com> > committer: Oleksandr Byelkin <sanja(a)mariadb.com> > timestamp: 2019-05-14 14:01:15 +0200 > message: > > MDEV-16932: ASAN heap-use-after-free in my_charlen_utf8 / > my_well_formed_char_length_utf8 on 2nd execution of SP with ALTER > trying to add bad CHECK > > In case of error the SP can be executed without re-comilation and so > will reuse constructed constaint name, so the name should be allocated > in the statement memory. Is that right? You're saying that the "unique constraint name" will be generated once and reused? But on the second SP invocation is might no longer be unique. I'd think that `ALTER TABLE t1 ADD CONSTRAINT CHECK (x > 0)` should add a new constraint, not "add a new contraint with the name that used to be unique when the SP was invoked for the first time". > diff --git a/sql/sql_table.cc b/sql/sql_table.cc > index 3df877792cc..ecd5de3a09d 100644 > --- a/sql/sql_table.cc > +++ b/sql/sql_table.cc > @@ -4185,9 +4185,15 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, > while ((check= c_it++)) > { > if (!check->name.length) > + { > + Query_arena backup; > + Query_arena *arena= thd->activate_stmt_arena_if_needed(&backup); > make_unique_constraint_name(thd, &check->name, > &alter_info->check_constraint_list, > &nr); > + if (arena) > + thd->restore_active_arena(arena, &backup); > + } > { > /* Check that there's no repeating constraint names. */ > List_iterator_fast<Virtual_column_info> Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

1 0

Re: [Maria-developers] [Commits] 66349551bf3: MDEV-16214: Incorrect plan taken by the optimizer , uses INDEX instead of ref access with ORDER BY
by Sergey Petrunia 19 May '19

19 May '19

Hi Varun, The patch is mostly fine. Please find some input on the comments / coding style below. Ok to push after it is addressed. On Fri, May 17, 2019 at 01:44:36PM +0530, Varun wrote: > revision-id: 66349551bf3a266fbefaebfe044abd05108723c2 (mariadb-10.3.12-205-g66349551bf3) > parent(s): 3d56adbfac394b2b3ffd22a89fe7c2978ed9a505 > author: Varun Gupta > committer: Varun Gupta > timestamp: 2019-05-17 13:44:05 +0530 > message: > > MDEV-16214: Incorrect plan taken by the optimizer , uses INDEX instead of ref access with ORDER BY > > The issue in this case is that we take in account the estimates from quick keys instead of rec_per_key. > The estimates for quick keys are better than rec_per_key only if we have ref(const), so we need to check > that all keyparts in the ref key are of the type ref(const). > > --- > mysql-test/main/order_by.result | 57 +++++++++++++++++++++++++++++++++++++++++ > mysql-test/main/order_by.test | 37 ++++++++++++++++++++++++++ > sql/sql_select.cc | 42 ++++++++++++++++++++---------- > 3 files changed, 122 insertions(+), 14 deletions(-) ... > diff --git a/sql/sql_select.cc b/sql/sql_select.cc > index f36a68bc7ae..8cdcf3afc8b 100644 > --- a/sql/sql_select.cc > +++ b/sql/sql_select.cc > @@ -9986,31 +9986,35 @@ static bool create_ref_for_key(JOIN *join, JOIN_TAB *j, > j->ref.null_rejecting|= (key_part_map)1 << i; > keyuse_uses_no_tables= keyuse_uses_no_tables && !keyuse->used_tables; > /* > - Todo: we should remove this check for thd->lex->describe on the next > - line. With SHOW EXPLAIN code, EXPLAIN printout code no longer depends > - on it. However, removing the check caused change in lots of query > - plans! Does the optimizer depend on the contents of > - table_ref->key_copy ? If yes, do we produce incorrect EXPLAINs? > + We don't want to compute heavy expressions in EXPLAIN, an example would > + select * from t1 where t1.key=(select thats very heavy); > + > + (select thats very heavy) => is a constant here > + eg: (select avg(order_cost) from orders) => constant but expensive > */ > if (!keyuse->val->used_tables() && !thd->lex->describe) > { // Compare against constant > - store_key_item tmp(thd, > + store_key_item tmp(thd, > keyinfo->key_part[i].field, > key_buff + maybe_null, > maybe_null ? key_buff : 0, > keyinfo->key_part[i].length, > keyuse->val, > FALSE); > - if (unlikely(thd->is_fatal_error)) > - DBUG_RETURN(TRUE); > - tmp.copy(); > + if (unlikely(thd->is_fatal_error)) > + DBUG_RETURN(TRUE); > + tmp.copy(); > j->ref.const_ref_part_map |= key_part_map(1) << i ; > } > else > - *ref_key++= get_store_key(thd, > - keyuse,join->const_table_map, > - &keyinfo->key_part[i], > - key_buff, maybe_null); > + { > + *ref_key++= get_store_key(thd, > + keyuse,join->const_table_map, > + &keyinfo->key_part[i], > + key_buff, maybe_null); > + if (!keyuse->val->used_tables()) > + j->ref.const_ref_part_map |= key_part_map(1) << i ; > + } > /* > Remember if we are going to use REF_OR_NULL > But only if field _really_ can be null i.e. we force JT_REF > @@ -25256,6 +25260,8 @@ bool JOIN_TAB::save_explain_data(Explain_table_access *eta, > { > if (!(eta->ref_list.append_str(thd->mem_root, "const"))) > return 1; > + if (thd->lex->describe) > + key_ref++; This is very cryptic. Please provide an explanation, something like: create_ref_for_key() handles keypart=const equalities as follows: - non-EXPLAIN execution will copy the "const" to lookup tuple immediately and will not add an element to ref.key_copy - EXPLAIN will put an element into ref.key_copy. Since we've just printed "const" for it, we should skip it here. > } > else > { > @@ -26921,7 +26927,15 @@ test_if_cheaper_ordering(const JOIN_TAB *tab, ORDER *order, TABLE *table, > */ > if (ref_key >= 0 && ref_key != MAX_KEY && tab->type == JT_REF) > { > - if (table->quick_keys.is_set(ref_key)) > + /* > + For all the parts of the ref key we check if all of them belong > + to the type ref(const). This is done because if all parts of the ref > + key are of type ref(const), then we are sure that the estimates > + provides by quick keys is better than that provide by rec_per_key. The above is hard to read and has typos. Here's a shorter variant: If ref access uses keypart=const for all its key parts, and quick select uses the same # of key parts, then they are equivalent. Reuse #rows estimate from quick select as it is more precise. > + */ > + if (tab->ref.const_ref_part_map == make_prev_keypart_map(tab->ref.key_parts) && > + table->quick_keys.is_set(ref_key) && > + table->quick_key_parts[ref_key] == tab->ref.key_parts) Please fix indetation; One of the lines is too long. > refkey_rows_estimate= table->quick_rows[ref_key]; > else > { BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

[Maria-developers] FYI: usage of F1-F12 keys in mysql client
by Aleksey Midenkov 17 May '19

17 May '19

By adding something like this in your .inputrc: $if MYSQL "\033OP": "use information_schema; show tables;\n" "\033OQ": "use test;\n" "\033OR": "select * from " "\033OS": "show columns from " "\033[15~": "\033[17~": "\033[18~": "\033[19~": "\033[20~": "\033[21~": "\033[23~": "\033[24~": $endif you may have F1-F12 keys utilized. Different terminals have different codes (.inputrc doesn't support capability names unfortunately), so you'll have to find out your own codes via tack utiity. With strong will and enough patience it is possible to add Ctrl+, Alt+, Shift+ and their combinations as well. This works only with MDEV-17609 patch. -- All the best, Aleksey Midenkov @midenok

1 0

Re: [Maria-developers] MDEV-19284-19285v3
by Eugene Kosov 14 May '19

14 May '19

Hi, Alexander! Patch looks good to me. 13.05.2019, 17:03, "Alexander Barkov" <bar(a)mariadb.com>: > Hi Eugene, > > Please review the next round of the patch. > > Are any changes in tests still needed? > > Thanks! -- Eugene

1 0

Re: [Maria-developers] 7198e7c71fc: MDEV-15458 Segfault in heap_scan() upon UPDATE after ADD SYSTEM VERSIONING
by Sergei Golubchik 13 May '19

13 May '19

Hi, Aleksey! On Dec 23, Aleksey Midenkov wrote: > revision-id: 7198e7c71fc (versioning-1.0.6-82-g7198e7c71fc) > parent(s): 6be155757b7 > author: Aleksey Midenkov <midenok(a)gmail.com> > committer: Aleksey Midenkov <midenok(a)gmail.com> > timestamp: 2018-12-20 13:25:00 +0300 > message: > > MDEV-15458 Segfault in heap_scan() upon UPDATE after ADD SYSTEM VERSIONING > > * HA_EXTRA_REMEMBER_POS, HA_EXTRA_RESTORE_POS for HEAP storage engine > * Versioning tests support As far as I can see, there are two approaches to fixing this bug. And if we go the way of HA_EXTRA_REMEMBER_POS/HA_EXTRA_RESTORE_POS then your fix is good. But I'd still like to understand whether write_row is supposed to disturb the scan. Why should heap_write() overwrite info->current_ptr=pos and info->current_hash_ptr? Inserting a row doesn't happen at the "current position", so I don't quite understand why it modifies current_ptr at all. There is no functionality like "continue the scan from the last inserted row", right? It just doesn't make sense to me. Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

2 2

[Maria-developers] Welcome to GSoC 2019
by Vicențiu Ciorbaru 10 May '19

10 May '19

Hi! As you may have noticed a few days ago, you have received an email from Google that you have been accepted as part of Google Summer of Code. Congratulations on the effort to learn about the program, get involved and submit succesful proposals. The fun part begins now! If you have not done so already, please reach out to your mentor and seek further advice about your project. Each one of your projects can be implemented in multiple ways, some more complex than others. I strongly advise you to create a concrete plan based off of your proposal with your mentor and aim to stick to it. Your mentor can guide you to ensure you have a succesful GSoC. Remember that you are here to learn about Open Source, what it means to be part of an Open Source community, to interact with the community, all while becoming better programmers. Please take advantage of this opportunity and keep your technical discussions public, on Zulip https://mariadb.zulipchat.com, and on our developer mailing list https://launchpad.net/~maria-developers. By keeping your discussions public, you get the benefit of not only having your mentor see your questions but all of the MariaDB community. You never know when someone might lend a hand in case your mentor is not yet online. :) The mantra of our community should be that we are always welcoming to new people. Do not be afraid to ask questions. There are no bad or stupid or silly questions. The only key requirement is that you first make an effort to find the answer yourself and only afterwards seek outside help. If you can show a bit of involvement, the community will show you the same. I recommend that you agree with your mentor on some form of weekly progress reporting, via either a blog-post, an email, or something similar that can be publicly shared. Hope all of you will have a great GSoC! Vicențiu MariaDB Foundation GSoC 2019 Admin

1 0

Re: [Maria-developers] a4be420c920: MDEV-19235 MariaDB Server compiled for 128 Indexes crashes at startup
by Sergei Golubchik 09 May '19

09 May '19

Hi, Vladislav! Thanks. That's pretty much the solution I had in mind too :) A couple of comments on details below. I've only commented on sql_bitmap.h, everything else is fine. On May 08, Vladislav Vaintroub wrote: > revision-id: a4be420c920 (mariadb-10.4.4-25-ga4be420c920) > parent(s): ee4a2fef181 > author: Vladislav Vaintroub <wlad(a)mariadb.com> > committer: Vladislav Vaintroub <wlad(a)mariadb.com> > timestamp: 2019-04-17 17:17:06 +0100 > message: > > MDEV-19235 MariaDB Server compiled for 128 Indexes crashes at startup > > With MAX_INDEXIES=64(default), key_map=Bitmap<64> is just a wrapper around > ulonglong and thus "trivial" (can be bzero-ed, or memcpy-ed, and stays > valid still) > > With MAX_INDEXES=128, key_map = Bitmap<128> is not a "trivial" type > anymore. The implementation uses MY_BITMAP, and MY_BITMAP contains pointers > which make Bitmap invalid, when it is memcpy-ed/bzero-ed. > > The problem in 10.4 is that there are many new key_map members, inside TABLE > or KEY, and those are often memcopied and bzeroed > > The fix makes Bitmap "trivial", by inlining most of MY_BITMAP functionality. > pointers/heap allocations are not used anymore. > diff --git a/sql/sql_bitmap.h b/sql/sql_bitmap.h > index 705a8d169e1..74866336a1a 100644 > --- a/sql/sql_bitmap.h > +++ b/sql/sql_bitmap.h > @@ -25,73 +25,184 @@ > > #include <my_sys.h> > #include <my_bitmap.h> > +#include <my_bit.h> > > template <uint width> class Bitmap > { > - MY_BITMAP map; > +public: > uint32 buffer[(width+31)/32]; Why is it public? > public: > - Bitmap() { init(); } > + Bitmap():buffer() What does that do? > + { > + } > - Bitmap(const Bitmap& from) { *this=from; } > - explicit Bitmap(uint prefix_to_set) { init(prefix_to_set); } > + explicit Bitmap(uint prefix_to_set):buffer() > + { > + set_prefix(prefix_to_set); > + } > - void init() { my_bitmap_init(&map, buffer, width, 0); } > - void init(uint prefix_to_set) { init(); set_prefix(prefix_to_set); } > uint length() const { return width; } > - Bitmap& operator=(const Bitmap& map2) > - { > + void init() > + { > + clear_all(); > + } > + void init(uint prefix_to_set) > + { > init(); > - memcpy(buffer, map2.buffer, sizeof(buffer)); > - return *this; > + set_prefix(prefix_to_set); I wonder whether init() method is still needed. You've removed it from the constructor. And if there's any other code that uses it, I'd rather it'd be using clear_all(). > } > - void set_bit(uint n) { bitmap_set_bit(&map, n); } > + void set_bit(uint n) > + { > + DBUG_ASSERT(n < width); > + ((uchar*)buffer)[n / 8] |= (1 << (n & 7)); > + } > - void clear_bit(uint n) { bitmap_clear_bit(&map, n); } > + void clear_bit(uint n) > + { > + DBUG_ASSERT(n < width); > + ((uchar*)buffer)[n / 8] &= ~(1 << (n & 7)); > + } > - void set_prefix(uint n) { bitmap_set_prefix(&map, n); } > + void set_prefix(uint prefix_size) > + { > + set_if_smaller(prefix_size, width); > + uint prefix_bytes, prefix_bits, d; > + uchar* m = (uchar*)buffer; > + > + if ((prefix_bytes = prefix_size / 8)) > + memset(m, 0xff, prefix_bytes); > + m += prefix_bytes; > + if ((prefix_bits = prefix_size & 7)) > + { > + *(m++) = (1 << prefix_bits) - 1; > + // As the prefix bits are set, lets count this byte too as a prefix byte. > + prefix_bytes++; > + } > + if ((d = (width + 7) / 8 - prefix_bytes)) > + memset(m, 0, d); > + } > - void set_all() { bitmap_set_all(&map); } > + void set_all() > + { > + set_prefix(width); may be better to optimize it? memset(buffer, 0xff, sizeof(buffer) - 1); ((uchar*)buffer)[sizeof(buffer)-1] = last_byte_mask(width); > + } > - void clear_all() { bitmap_clear_all(&map); } > + void clear_all() > + { > + memset(buffer, 0x00, sizeof(buffer)); > + } > - void intersect(Bitmap& map2) { bitmap_intersect(&map, &map2.map); } > + void intersect(Bitmap & map2) > + { > + for (uint i = 0; i < array_elements(buffer); i++) > + buffer[i] &= map2.buffer[i]; > + } > - void intersect(ulonglong map2buff) > - { > - // Use a spearate temporary buffer, as bitmap_init() clears all the bits. > - ulonglong buf2; > - MY_BITMAP map2; > - > - my_bitmap_init(&map2, (uint32 *) &buf2, sizeof(ulonglong) * 8, 0); > - > - // Store the original bits. > - if (sizeof(ulonglong) >= 8) > - { > - int8store(const_cast<uchar *>(static_cast<uchar *> > - (static_cast<void *>(&buf2))), > - map2buff); > - } > - else > - { > - DBUG_ASSERT(sizeof(buffer) >= 4); > - int4store(const_cast<uchar *>(static_cast<uchar *> > - (static_cast<void *>(&buf2))), > - static_cast<uint32>(map2buff)); > - } > - bitmap_intersect(&map, &map2); > - } > + void intersect_and_pad(ulonglong map2buff, bool pad_with_ones) make it private? > + { > + compile_time_assert(sizeof(ulonglong) == 8); > + uint32 tmp[2]; > + int8store(tmp, map2buff); > + if (array_elements(buffer) > 2 && pad_with_ones) > + set_all(); really? I'd expect that if pad_with_ones==true, you simply don't need to do anything to the rest of the buffer. > + > + buffer[0] &= tmp[0]; > + if (array_elements(buffer) > 1) > + buffer[1] &= tmp[1]; > + > + if (array_elements(buffer) > 2 && !pad_with_ones) > + memset((char*)buffer + 8, 0 , sizeof(buffer) - 8); > + } > + void intersect(ulonglong map2buff) > + { > + intersect_and_pad(map2buff, 0); > + } > /* Use highest bit for all bits above sizeof(ulonglong)*8. */ > void intersect_extended(ulonglong map2buff) > { > - intersect(map2buff); > - if (map.n_bits > sizeof(ulonglong) * 8) > - bitmap_set_above(&map, sizeof(ulonglong), > - MY_TEST(map2buff & (1LL << (sizeof(ulonglong) * 8 - 1)))); > + intersect_and_pad(map2buff, (map2buff & (1ULL << 63))); > } > - void subtract(Bitmap& map2) { bitmap_subtract(&map, &map2.map); } > + void subtract(Bitmap & map2) > + { > + for (size_t i = 0; i < array_elements(buffer); i++) > + buffer[i] &= ~(map2.buffer[i]); > + } > - void merge(Bitmap& map2) { bitmap_union(&map, &map2.map); } > + void merge(Bitmap & map2) > + { > + for (size_t i = 0; i < array_elements(buffer); i++) > + buffer[i] |= map2.buffer[i]; > + } > - bool is_set(uint n) const { return bitmap_is_set(&map, n); } > + bool is_set(uint n) const > + { > + DBUG_ASSERT(n < width); > + return ((uchar*)buffer)[n / 8] & (1 << (n & 7)); > + } > - bool is_prefix(uint n) const { return bitmap_is_prefix(&map, n); } > + bool is_prefix(uint prefix_size) const > + { > + uint prefix_mask = last_byte_mask(prefix_size); > + uchar* m = (uchar*)buffer; > + uchar* end_prefix = m + (prefix_size - 1) / 8; > + uchar* end; > + DBUG_ASSERT(prefix_size <= width); > + > + /* Empty prefix is always true */ > + if (!prefix_size) > + return true; > + > + while (m < end_prefix) > + if (*m++ != 0xff) > + return false; > + > + end = ((uchar*)buffer) + (width + 7) / 8 - 1; > + if (m == end) > + return ((*m & last_byte_mask(width)) == prefix_mask); > + > + if (*m != prefix_mask) > + return false; > + > + while (++m < end) > + if (*m != 0) > + return false; > + return ((*m & last_byte_mask(width)) == 0); > + } > - bool is_clear_all() const { return bitmap_is_clear_all(&map); } > + bool is_clear_all() const > + { > + for (size_t i= 0; i < array_elements(buffer); i++) > + if (buffer[i]) > + return false; > + return true; > + } > - bool is_set_all() const { return bitmap_is_set_all(&map); } > + bool is_set_all() const > + { > + if (width == sizeof(buffer) * 8) > + { > + for (size_t i = 0; i < array_elements(buffer); i++) > + if (buffer[i] != 0xFFFFFFFFU) > + return false; > + return true; > + } > + else > + return is_prefix(width); this can also be optimized to not use complex is_prefix(), but I don't think is_set_all() is used as often as set_all(), so it shouldn't matter as much. > + } > - bool is_subset(const Bitmap& map2) const { return bitmap_is_subset(&map, &map2.map); } > + bool is_subset(const Bitmap & map2) const > + { > + for (size_t i= 0; i < array_elements(buffer); i++) > + if (buffer[i] & ~(map2.buffer[i])) > + return false; > + return true; > + } > - bool is_overlapping(const Bitmap& map2) const { return bitmap_is_overlapping(&map, &map2.map); } > + bool is_overlapping(const Bitmap & map2) const > + { > + for (size_t i = 0; i < array_elements(buffer); i++) > + if (buffer[i] & map2.buffer[i]) > + return true; > + return false; > + } > - bool operator==(const Bitmap& map2) const { return bitmap_cmp(&map, &map2.map); } > + bool operator==(const Bitmap & map2) const > + { > + return memcmp(buffer, map2.buffer, sizeof(buffer)) == 0; > + } > - bool operator!=(const Bitmap& map2) const { return !(*this == map2); } > + bool operator!=(const Bitmap & map2) const > + { > + return !(*this == map2); > + } > char *print(char *buf) const > { > char *s=buf; Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

2 2

Re: [Maria-developers] [Commits] 0ff8b73b60f: MDEV-15777:Support Early NULLs filtering-like restrictions in the range optimizer
by Sergey Petrunia 06 May '19

06 May '19

Hi Varun, Please finally find the review below. The patch is almost there I think, but there are some minor adjustments that still need to be made. > revision-id: 0ff8b73b60fae40556f91d3959f20bf3cf182b28 (mariadb-10.3.0-947-g0ff8b73b60f) > parent(s): 15419a558370aeed9521b498c34d50f20a8d47a5 > author: Varun Gupta > committer: Varun Gupta > timestamp: 2018-05-15 13:53:11 +0530 > message: > > MDEV-15777:Support Early NULLs filtering-like restrictions in the range optimizer > > --- > mysql-test/main/mdev15777.result | 455 +++++++++++++++++++++++++++++++++++++++ > mysql-test/main/mdev15777.test | 138 ++++++++++++ > sql/opt_range.cc | 125 ++++++++++- > sql/opt_range.h | 2 + > sql/sql_select.cc | 22 +- > sql/table.cc | 1 + > sql/table.h | 6 + > 7 files changed, 745 insertions(+), 4 deletions(-) > > diff --git a/mysql-test/main/mdev15777.result b/mysql-test/main/mdev15777.result > new file mode 100644 > index 00000000000..549ac08b91f > --- /dev/null > +++ b/mysql-test/main/mdev15777.result > @@ -0,0 +1,455 @@ > +# Test added to check that null filtering is used by range optimizer > +create table ten(a int); > +insert into ten values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); > +create table one_k(a int); > +insert into one_k select A.a + B.a* 10 + C.a * 100 from ten A, ten B, ten C; > +create table one_m(a int); > +insert into one_m select A.a + B.a* 1000 from one_k A, one_k B; The testcases are extremely confusing. * Do you really need a table with one millon rows to demonstrate the issue? * Why use examples with subqueries (which are converted into join anyway)? - if we want to verify that this optimization works together with subquery-to-semi-join conversion, let's start with a join example, and then construct a subquery which is converted to the query. * The testcases produce a lot of EXPLAIN outputs, and there is no explanation provided what one should expect. Please create a testcase: - of reasonable size (I think tables of 10-20K rows should suffice) - start from a join query - Like in other testcases, tables should be named, t1,t2, ... - I assume the query plan should be: range access on the first table (constructed from a NOT NULL predicate), ref access for the second one. - The testcase should have '--echo #' comments explaining what is relevant in the query plan > +delete from one_m where a=0 limit 1; > +create table t1 ( > +id int(10) unsigned NOT NULL AUTO_INCREMENT, > +filler varchar(100), > +subset_id int(11) DEFAULT NULL, > +PRIMARY KEY (id), > +KEY t1_subset_id (subset_id) > +); > +create table t1_subsets ( ... > diff --git a/sql/opt_range.cc b/sql/opt_range.cc > index 8422917065f..ae06b899ac9 100644 > --- a/sql/opt_range.cc > +++ b/sql/opt_range.cc > @@ -155,6 +155,7 @@ class SEL_IMERGE; > #define CLONE_KEY1_MAYBE 1 > #define CLONE_KEY2_MAYBE 2 > #define swap_clone_flag(A) ((A & 1) << 1) | ((A & 2) >> 1) > +#define FT_KEYPART (MAX_FIELDS+10) > > > /* > @@ -2400,6 +2401,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, > { > uint idx; > double scan_time; > + Item *null_rejecting_conds= NULL; > DBUG_ENTER("SQL_SELECT::test_quick_select"); > DBUG_PRINT("enter",("keys_to_use: %lu prev_tables: %lu const_tables: %lu", > (ulong) keys_to_use.to_ulonglong(), (ulong) prev_tables, > @@ -2423,6 +2425,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, > read_time= (double) records + scan_time + 1; // Force to use index > > possible_keys.clear_all(); > + null_rejecting_conds= head->null_rejecting_conds; > > DBUG_PRINT("info",("Time to scan table: %g", read_time)); > > @@ -2431,7 +2434,7 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, > { > uchar buff[STACK_BUFF_ALLOC]; > MEM_ROOT alloc; > - SEL_TREE *tree= NULL; > + SEL_TREE *tree= NULL, *not_null_cond_tree= NULL; > KEY_PART *key_parts; > KEY *key_info; > PARAM param; > @@ -2540,6 +2543,12 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, > TRP_GROUP_MIN_MAX *group_trp; > double best_read_time= read_time; > Please add a comment saying that as far as range optimizer is concerned, null_rejecting_conds is just an extra condition on this table. (That is, we don't care if it has NOT NULL conditions or something else). > + if (null_rejecting_conds) > + { > + not_null_cond_tree= null_rejecting_conds->get_mm_tree(&param, > + &null_rejecting_conds); > + } > + > if (cond) > { > if ((tree= cond->get_mm_tree(&param, &cond))) > @@ -2558,6 +2567,13 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, > tree= NULL; > } > } > + if (not_null_cond_tree) > + { > + if (!tree) > + tree= not_null_cond_tree; > + else > + tree= tree_and(&param, tree, not_null_cond_tree); This if-else logic is redundant, as it's already present in tree_and(). Can just have this: tree= tree_and(tree, not_null_cond_tree); > + } > > /* > Try to construct a QUICK_GROUP_MIN_MAX_SELECT. > @@ -14643,6 +14659,113 @@ void QUICK_GROUP_MIN_MAX_SELECT::add_keys_and_lengths(String *key_names, > add_key_and_length(key_names, used_lengths, &first); > } > > +inline void add_cond(THD *thd, Item **e1, Item *e2) > +{ > + if (*e1) > + { > + if (!e2) > + return; > + Item *res; > + if ((res= new (thd->mem_root) Item_cond_and(thd, *e1, e2))) > + { > + res->fix_fields(thd, 0); > + res->update_used_tables(); > + *e1= res; > + } > + } > + else > + *e1= e2; > +} > + > +/* > + Create null rejecting conditions for a table, for all the equalites > + present in the WHERE clause of a query. > + > + SYNOPSIS > + make_null_rejecting_conds() > + @param TABLE - Keys of this table will participate in null > + rejecting conditions > + @param keyuse_array - array that has all the equalites of the > + WHERE clasuse > + > + DESCRIPTION > + This function creates null rejecting conditions for a table. These > + conditions are created to do range analysis on them , the conditions > + are of the form tbl.key.keypart IS NOT NULL. > + > + IMPLEMENTATION > + Lookup in the keyuse array to check if it has equalites that belong > + to the given table. If yes then find out if the conditions are null > + rejecting and accordingly create all the condition for the keys of a > + given table and AND them. > + > + > + RETURN > + NOT NULL - Found null rejecting conditions for the given table > + NULL - No null rejecting conditions for the given table > +*/ > + > +void make_null_rejecting_conds(THD *thd, TABLE *table, > + DYNAMIC_ARRAY *keyuse_array, key_map *const_keys) ^ Please fix identation. > +{ > + KEY *keyinfo; > + Item *cond= NULL; > + KEYUSE* keyuse; > + > + /* > + The null rejecting conds added will be on the keypart of a key, so for > + that we need the table to atleast have a key. > + */ please fix typos/wording in the above comment. > + if (!table->s->keys) > + return ; > + if (table->null_rejecting_conds) > + return; > + > + for(uint i=0; i < keyuse_array->elements; i++) > + { This loop does excessive work. Check out how best_access_path() examines the KEYUSE array. Key points - the array is sorted by table, index_nr, key_part_nr. - JOIN_TAB::keyuse points to the first element that refers to the table described by the JOIN_TAB - The array may have "duplicates", especially when multiple equalities are present: KEYUSE(t.keyXpartY = other_table.colX) KEYUSE(t.keyXpartY = other_table.colY) KEYUSE(t.keyXpartY = some_expr) ... Since the elements are sorted by keyXpartY, it is trival to avoid generating multiple "t.keyXpartY IS NOT NULL" for consecutive KEYUSEs. I would go even further: the same "t.keyXpartY" can be part of multiple indexes (and so its KEYUSE objects will not be adjacent). Can we use a table's column bitmap to avoid generating duplicate IS NOT NULL objects? > + keyuse= (KEYUSE*)dynamic_array_ptr(keyuse_array, i); > + if (keyuse->table == table) > + { > + /* > + No null rejecting conds for a hash key orr full-text keys > + */ please fix typos/wording in the above comment. > + if (keyuse->key == MAX_KEY || keyuse->keypart == FT_KEYPART) > + continue; > + keyinfo= keyuse->table->key_info+keyuse->key; > + Field *field= keyinfo->key_part[keyuse->keypart].field; > + > + /* > + No need to add null-rejecting condition if we have a > + keyuse element as > + - table.key.keypart= const > + - (table.key.keypart= tbl.otherfield or table.key.keypart IS NULL) > + - table.key.keypart IS NOT NULLABLE > + */ > + > + if (keyuse->val->const_item() > + || !(keyuse->null_rejecting && field->maybe_null()) Please remove the brackets: !(A && B) -> !A || !B > + || keyuse->optimize & KEY_OPTIMIZE_REF_OR_NULL) Please also put the '||' at the end of the lines, like it is done in the rest of the code. > + continue; > + > + Item_field *field_item= new (thd->mem_root)Item_field(thd, field); > + Item* not_null_item= new (thd->mem_root)Item_func_isnotnull(thd, > + field_item); Please add error handling (just return from the function if we failed to allocate any of the above) > + > + /* > + adding the key to const keys as we have the condition > + as key.keypart IS NOT NULL > + */ > + > + const_keys->set_bit(keyuse->key); > + not_null_item->fix_fields(thd, 0); > + not_null_item->update_used_tables(); > + add_cond(thd, &cond, not_null_item); > + } > + } > + table->null_rejecting_conds= cond; > + return; Please remove the above 'return;' as it is redundant and makes one suspect a merge error > +} > + > > #ifndef DBUG_OFF > > diff --git a/sql/opt_range.h b/sql/opt_range.h > index bd85a12d4a1..894d46a892c 100644 > --- a/sql/opt_range.h > +++ b/sql/opt_range.h > @@ -1728,6 +1728,8 @@ bool calculate_cond_selectivity_for_table(THD *thd, TABLE *table, Item **cond); > bool prune_partitions(THD *thd, TABLE *table, Item *pprune_cond); > #endif > void store_key_image_to_rec(Field *field, uchar *ptr, uint len); > +void make_null_rejecting_conds(THD *thd, TABLE *table, > + DYNAMIC_ARRAY *keyuse_array, key_map *const_keys); Please fix identation. > > extern String null_string; > > diff --git a/sql/sql_select.cc b/sql/sql_select.cc > index f658b78c33c..4aceb333340 100644 > --- a/sql/sql_select.cc > +++ b/sql/sql_select.cc > @@ -4805,6 +4805,9 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list, > add_group_and_distinct_keys(join, s); > > s->table->cond_selectivity= 1.0; > + > + make_null_rejecting_conds(join->thd, s->table, > + keyuse_array, &s->const_keys); > > /* > Perform range analysis if there are keys it could use (1). > @@ -5352,15 +5356,24 @@ add_key_field(JOIN *join, > If the condition has form "tbl.keypart = othertbl.field" and > othertbl.field can be NULL, there will be no matches if othertbl.field > has NULL value. > + > + The field KEY_FIELD::null_rejecting is set to TRUE if we have both > + the left and right hand side of the equality are NULLABLE > + This is NOT what is happening. null_rejecting is set to true, if *either* left or right-hand side of the equality is nullable. Please fix the comment. Please also fix the comment in KEY_FIELD::null_rejecting to reflect that. > We use null_rejecting in add_not_null_conds() to add > 'othertbl.field IS NOT NULL' to tab->select_cond. > + > + We use null_rejecting in make_null_rejecting_conds() to add > + tbl.keypart IS NOT NULL so we can do range analysis on this condition > + Please join the above two sentences together: null_rejecting is used - by add_not_null_conds(), to add "othertbl.field IS NOT NULL" to othertbl's tab->select_cond. (This is called "Early NULLs filtering") - by make_null_rejecting_conds(), to provide range optimizer with additional "tbl.keypart IS NOT NULL" condition. > */ > { > Item *real= (*value)->real_item(); > if (((cond->functype() == Item_func::EQ_FUNC) || > (cond->functype() == Item_func::MULT_EQUAL_FUNC)) && > - (real->type() == Item::FIELD_ITEM) && > + (((real->type() == Item::FIELD_ITEM) && > ((Item_field*)real)->field->maybe_null()) > + ||(field->maybe_null()))) ^Please fix the identation. > (*key_fields)->null_rejecting= true; > else > (*key_fields)->null_rejecting= false; > @@ -9813,7 +9826,10 @@ static bool create_ref_for_key(JOIN *join, JOIN_TAB *j, > uint maybe_null= MY_TEST(keyinfo->key_part[i].null_bit); > j->ref.items[i]=keyuse->val; // Save for cond removal > j->ref.cond_guards[i]= keyuse->cond_guard; > - if (keyuse->null_rejecting) > + Item *real= (keyuse->val)->real_item(); > + if (keyuse->null_rejecting && > + (real->type() == Item::FIELD_ITEM) && > + ((Item_field*)real)->field->maybe_null()) ^ Please fix identation > j->ref.null_rejecting|= (key_part_map)1 << i; > keyuse_uses_no_tables= keyuse_uses_no_tables && !keyuse->used_tables; > /* > @@ -18538,7 +18554,7 @@ free_tmp_table(THD *thd, TABLE *entry) > DBUG_ASSERT(entry->pos_in_table_list->table == entry); > entry->pos_in_table_list->table= NULL; > } > - > + entry->null_rejecting_conds= NULL; What is this for? As far as I understand we are freeing the TABLE object > free_root(&own_root, MYF(0)); /* the table is allocated in its own root */ > thd_proc_info(thd, save_proc_info); > > diff --git a/sql/table.cc b/sql/table.cc > index a6e445d0d2e..097fee465de 100644 > --- a/sql/table.cc > +++ b/sql/table.cc > @@ -4597,6 +4597,7 @@ void TABLE::init(THD *thd, TABLE_LIST *tl) > created= TRUE; > cond_selectivity= 1.0; > cond_selectivity_sampling_explain= NULL; > + null_rejecting_conds= NULL; > #ifdef HAVE_REPLICATION > /* used in RBR Triggers */ > master_had_triggers= 0; > diff --git a/sql/table.h b/sql/table.h > index 6fd3f219914..50d47899b49 100644 > --- a/sql/table.h > +++ b/sql/table.h > @@ -1356,6 +1356,12 @@ struct TABLE > SplM_opt_info *spl_opt_info; > key_map keys_usable_for_splitting; > > + /* > + Null rejecting conds added for all tables so we can do range analysis > + on these conditions > + */ "for all tables"? Please change the comment to something like: "NULL-rejecting conditions for this table. They are created from eq_ref access, and used by the range optimizer". > + Item* null_rejecting_conds; > + > void init(THD *thd, TABLE_LIST *tl); > bool fill_item_list(List<Item> *item_list) const; > void reset_item_list(List<Item> *item_list, uint skip) const; BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

[Maria-developers] [MDEV-18684] [5.5] memory leak
by Michal Schorm 03 May '19

03 May '19

Hello, can somebody please take a brief look to atleast evaulate, how hard it would be to backoport the patch form the latest library versions? https://jira.mariadb.org/browse/MDEV-18684 Thank you -- Michal Schorm Software Engineer Core Services - Databases Team Red Hat --

1 0

Re: [Maria-developers] [Commits] 5e23f561984: MDEV-18741: Optimizer trace: multi-part key ranges are printed incorrectly
by Sergey Petrunia 03 May '19

03 May '19

Hi Varun, Please find my review comments below. On Fri, Apr 05, 2019 at 02:59:17PM +0530, Varun wrote: > revision-id: 5e23f56198493ed19ac270759963f615324b2c49 (mariadb-10.4.3-162-g5e23f561984) > parent(s): 02d9b048a2ab549a3227a81e15ff2f8c45562a65 > author: Varun Gupta > committer: Varun Gupta > timestamp: 2019-04-05 14:58:39 +0530 > message: > > MDEV-18741: Optimizer trace: multi-part key ranges are printed incorrectly > > Changed the function append_range_all_keyparts to use sel_arg_range_seq_init / sel_arg_range_seq_next to produce ranges. > Also adjusted to print format for the ranges, now the ranges are printed as: > (keypart1_min, keypart2_min,..) OP (keypart1_name,keypart2_name, ..) OP (keypart1_max,keypart2_max, ..) > > Also added more tests for range and index merge access for optimizer trace > > --- > mysql-test/main/opt_trace.result | 276 ++++++++++- > mysql-test/main/opt_trace.test | 57 +++ > mysql-test/main/opt_trace_index_merge.result | 509 ++++++++++++++++++++- > mysql-test/main/opt_trace_index_merge.test | 112 +++++ > .../main/opt_trace_index_merge_innodb.result | 6 +- > sql/opt_range.cc | 281 ++++++------ > sql/opt_range.h | 7 +- > 7 files changed, 1079 insertions(+), 169 deletions(-) ... > +explain select * from t1 force index(a_b_c) where a between 1 and 4 and b < 50; > +id select_type table type possible_keys key key_len ref rows Extra > +1 SIMPLE t1 range a_b_c a_b_c 8 NULL 4 Using index condition > +select JSON_DETAILED(JSON_EXTRACT(trace, '$**.analyzing_range_alternatives')) from INFORMATION_SCHEMA.OPTIMIZER_TRACE; > +JSON_DETAILED(JSON_EXTRACT(trace, '$**.analyzing_range_alternatives')) > +[ > + > + { > + "range_scan_alternatives": > + [ > + > + { > + "index": "a_b_c", > + "ranges": > + [ > + "(1) <= (a) < (4,50)" In the example above, only the name of the first key part is printed, even if the second endpoint uses two. As far as I understand the code, just using range->start_key.keypart_map | range->end_key.keypart_map in print_keyparts_name will be sufficient to fix this. > + ], > + "rowid_ordered": false, > + "using_mrr": false, > + "index_only": false, > + "rows": 4, > + "cost": 6.2648, > + "chosen": true > + } > + ], > + "analyzing_roworder_intersect": > + { > + "cause": "too few roworder scans" > + }, > + "analyzing_index_merge_union": > + [ > + ] > + } > +] > diff --git a/sql/opt_range.cc b/sql/opt_range.cc > index 5ab3d70214d..1827e2fcaf2 100644 > --- a/sql/opt_range.cc > +++ b/sql/opt_range.cc > @@ -431,16 +431,18 @@ static int and_range_trees(RANGE_OPT_PARAM *param, > static bool remove_nonrange_trees(RANGE_OPT_PARAM *param, SEL_TREE *tree); > > static void print_key_value(String *out, const KEY_PART_INFO *key_part, > - const uchar *key); > + const uchar* key, uint length); > +static void print_keyparts_name(String *out, const KEY_PART_INFO *key_part, > + uint n_keypart, key_part_map keypart_map); > > static void append_range_all_keyparts(Json_writer_array *range_trace, > - String *range_string, > - String *range_so_far, const SEL_ARG *keypart, > + PARAM param, uint idx, > + SEL_ARG *keypart, > const KEY_PART_INFO *key_parts); > > static > -void append_range(String *out, const KEY_PART_INFO *key_parts, > - const uchar *min_key, const uchar *max_key, const uint flag); > +void append_range(String *out, const KEY_PART_INFO *key_part, > + KEY_MULTI_RANGE *range, uint n_key_parts); Why is this function called append_() while other functions that print stuff into strings are called print_key_value, print_keyparts_name ? The name "append_range_all_keyparts" is also confusing. The function prints a sequence of ranges. One can say it "appends" them to the trace, but that verb is not used to refer to this action anywhere else. Functions that print something into trace are called trace_XXX(), e.g. TRP_RANGE::trace_basic_info(), trace_plan_prefix(), etc. I think trace_ranges() would be a better name here. What do you think? > > > /* > @@ -2273,10 +2275,7 @@ void TRP_RANGE::trace_basic_info(const PARAM *param, > // TRP_RANGE should not be created if there are no range intervals > DBUG_ASSERT(key); > > - String range_info; > - range_info.length(0); > - range_info.set_charset(system_charset_info); > - append_range_all_keyparts(&trace_range, NULL, &range_info, key, key_part); > + append_range_all_keyparts(&trace_range, *param, key_idx, key, key_part); > } > > > @@ -2489,10 +2488,8 @@ void TRP_GROUP_MIN_MAX::trace_basic_info(const PARAM *param, > // can have group quick without ranges > if (index_tree) > { > - String range_info; > - range_info.set_charset(system_charset_info); > - append_range_all_keyparts(&trace_range, NULL, &range_info, index_tree, > - key_part); > + append_range_all_keyparts(&trace_range, *param, param_idx, > + index_tree, key_part); > } > } > > @@ -6398,20 +6395,9 @@ void TRP_ROR_INTERSECT::trace_basic_info(const PARAM *param, > trace_isect_idx.add("rows", (*cur_scan)->records); > > Json_writer_array trace_range(thd, "ranges"); > - for (const SEL_ARG *current= (*cur_scan)->sel_arg->first(); current; > - current= current->next) > - { > - String range_info; > - range_info.set_charset(system_charset_info); > - for (const SEL_ARG *part= current; part; > - part= part->next_key_part ? part->next_key_part : nullptr) > - { > - const KEY_PART_INFO *cur_key_part= key_part + part->part; > - append_range(&range_info, cur_key_part, part->min_value, > - part->max_value, part->min_flag | part->max_flag); > - } > - trace_range.add(range_info.ptr(), range_info.length()); > - } > + > + append_range_all_keyparts(&trace_range, *param, (*cur_scan)->idx, > + (*cur_scan)->sel_arg->first(), key_part); What is this for? I think this is not needed (at best). Range enumeration should start from the tree root, not from the leftmost element. > } > } > ... > @@ -15775,60 +15763,39 @@ void append_range(String *out, const KEY_PART_INFO *key_part, > > Add ranges to the trace > For ex: > - query: select * from t1 where a=2 ; > - and we have an index on a , so we create a range > - 2 <= a <= 2 > + lets say we have an index a_b(a,b) > + query: select * from t1 where a=2 and b=4 ; > + so we create a range: > + (2,4) <= (a,b) <= (2,4) > this is added to the trace > */ > > static void append_range_all_keyparts(Json_writer_array *range_trace, > - String *range_string, > - String *range_so_far, const SEL_ARG *keypart, > + PARAM param, uint idx, Is it intentional that PARAM is passed by value? What for? > + SEL_ARG *keypart, > const KEY_PART_INFO *key_parts) > { > - > - DBUG_ASSERT(keypart); > - DBUG_ASSERT(keypart && keypart != &null_element); > - > - // Navigate to first interval in red-black tree > + SEL_ARG_RANGE_SEQ seq; > + KEY_MULTI_RANGE range; > + range_seq_t seq_it; > + uint flags= 0; > + RANGE_SEQ_IF seq_if = {NULL, sel_arg_range_seq_init, > + sel_arg_range_seq_next, 0, 0}; > + KEY *keyinfo= param.table->key_info + param.real_keynr[idx]; > + uint n_key_parts= param.table->actual_n_key_parts(keyinfo); > + seq.keyno= idx; > + seq.real_keyno= param.real_keynr[idx]; > + seq.param= &param; > + seq.start= keypart; > const KEY_PART_INFO *cur_key_part= key_parts + keypart->part; > - const SEL_ARG *keypart_range= keypart->first(); > - const size_t save_range_so_far_length= range_so_far->length(); > - > + seq_it= seq_if.init((void *) &seq, 0, flags); > > - while (keypart_range) > + while (!seq_if.next(seq_it, &range)) > { > - // Append the current range predicate to the range String > - switch (keypart->type) > - { > - case SEL_ARG::Type::KEY_RANGE: > - append_range(range_so_far, cur_key_part, keypart_range->min_value, > - keypart_range->max_value, > - keypart_range->min_flag | keypart_range->max_flag); > - break; > - case SEL_ARG::Type::MAYBE_KEY: > - range_so_far->append("MAYBE_KEY"); > - break; > - case SEL_ARG::Type::IMPOSSIBLE: > - range_so_far->append("IMPOSSIBLE"); > - break; > - default: > - DBUG_ASSERT(false); > - break; > - } > - > - if (keypart_range->next_key_part && > - keypart_range->next_key_part->part == > - keypart_range->part + 1 && > - keypart_range->is_singlepoint()) > - { > - append_range_all_keyparts(range_trace, range_string, range_so_far, > - keypart_range->next_key_part, key_parts); > - } > - else > - range_trace->add(range_so_far->c_ptr_safe(), range_so_far->length()); > - keypart_range= keypart_range->next; > - range_so_far->length(save_range_so_far_length); > + String range_info; > + range_info.set_charset(system_charset_info); Can you use "StringBuffer<128> tmp(system_charset_info);" like you're already doing in other places of this patch? > + append_range(&range_info, cur_key_part, &range, n_key_parts); > + range_trace->add(range_info.c_ptr_safe(), range_info.length()); > } > } > BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

Re: [Maria-developers] [Commits] bf951046e8d: MDEV-15296: wrong result with window function inside a subquery
by Oleksandr Byelkin 30 Apr '19

30 Apr '19

Hi, Varun! This bug shows us whole class of the same problem in the code which slip of our scope during window function development. so please check and fix them (one or separate change-set as you wish). See comment below. Am 29.04.19 um 16:40 schrieb Varun: > revision-id: bf951046e8df13702ee56e4e269ce5076e15407f (mariadb-10.2.23-99-gbf951046e8d) > parent(s): 092602ac9b650f921ec5380866d17d740f0eedb4 > author: Varun Gupta > committer: Varun Gupta > timestamp: 2019-04-29 20:07:47 +0530 > message: > > MDEV-15296: wrong result with window function inside a subquery > > Window Functions were treated as a constant in a dependent tables less subquery. > Made sure that the behaviour of window functions is same as the aggregate function > for dependent tables less subquery. > > --- > mysql-test/r/win.result | 17 +++++++++++++++++ > mysql-test/t/win.test | 10 ++++++++++ > sql/item_subselect.cc | 1 + > 3 files changed, 28 insertions(+) > > diff --git a/mysql-test/r/win.result b/mysql-test/r/win.result > index 0ddffc551dc..b86f5b1fc16 100644 > --- a/mysql-test/r/win.result > +++ b/mysql-test/r/win.result > @@ -3518,5 +3518,22 @@ rank() OVER (ORDER BY 1) ROW_NUMBER() OVER (ORDER BY (EXPORT_SET(5,'Y','N',',',4 > 1 3 > drop table t1; > # > +# MDEV-15296: wrong result with window function inside a subquery > +# > +CREATE TABLE t1(i INT); > +INSERT INTO t1 VALUES (1), (2); > +EXPLAIN EXTENDED SELECT (SELECT SUM(i) OVER (partition BY i)) FROM t1; > +id select_type table type possible_keys key key_len ref rows filtered Extra > +1 PRIMARY t1 ALL NULL NULL NULL NULL 2 100.00 > +2 DEPENDENT SUBQUERY NULL NULL NULL NULL NULL NULL NULL NULL No tables used > +Warnings: > +Note 1276 Field or reference 'test.t1.i' of SELECT #2 was resolved in SELECT #1 > +Note 1003 select <expr_cache><`test`.`t1`.`i`>((select sum(`test`.`t1`.`i`) over ( partition by `test`.`t1`.`i`))) AS `(SELECT SUM(i) OVER (partition BY i))` from `test`.`t1` > +SELECT (SELECT SUM(i) OVER (partition BY i)) FROM t1; > +(SELECT SUM(i) OVER (partition BY i)) > +1 > +2 > +drop table t1; > +# > # End of 10.2 tests > # > diff --git a/mysql-test/t/win.test b/mysql-test/t/win.test > index fd31e9d4bd9..6cc681cace4 100644 > --- a/mysql-test/t/win.test > +++ b/mysql-test/t/win.test > @@ -2265,6 +2265,16 @@ insert into t1 values (1),(2),(3); > SELECT rank() OVER (ORDER BY 1), ROW_NUMBER() OVER (ORDER BY (EXPORT_SET(5,'Y','N',',',4))) FROM t1; > drop table t1; > > +--echo # > +--echo # MDEV-15296: wrong result with window function inside a subquery > +--echo # > + > +CREATE TABLE t1(i INT); > +INSERT INTO t1 VALUES (1), (2); > +EXPLAIN EXTENDED SELECT (SELECT SUM(i) OVER (partition BY i)) FROM t1; > +SELECT (SELECT SUM(i) OVER (partition BY i)) FROM t1; > +drop table t1; > + > --echo # > --echo # End of 10.2 tests > --echo # > diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc > index 8cff8f3a5c4..95482364225 100644 > --- a/sql/item_subselect.cc > +++ b/sql/item_subselect.cc > @@ -1138,6 +1138,7 @@ Item_singlerow_subselect::select_transformer(JOIN *join) > !select_lex->table_list.elements && > select_lex->item_list.elements == 1 && > !select_lex->item_list.head()->with_sum_func && > + !select_lex->item_list.head()->with_window_func && This case is definitely OK, but I also asked you to check other cases with with_sum_func, I do not see traces of it here so I did and it is my findings: Item_cache_wrapper::Item_cache_wrapper do not transfer with_window_func, is it impossible to have it here? (then better to put ASSERT) (same Item_cache_wrapper::get_tmp_table_item) Item_in_optimizer::fix_left, Item_in_optimizer::fix_fields, Item_func_interval::fix_length_and_dec do not transfer the flag also. ASSERT in Item_equal::fix_fields Item_func::get_tmp_table_item, Item_subselect::get_tmp_table_item impossible? than better put an ASSERT. Item_func constructors do not transfer the flag. Item_allany_subselect::cleanup reset. Item_singlerow_subselect::select_transformer, Item_in_subselect::single_value_transformer, Item_allany_subselect::transform_into_max_min, Item_exists_subselect::exists2in_processor, check_and_do_in_subquery_rewrites I doubts that window function allows the transformation is aggregate prohibit it. Item_in_subselect::create_single_in_to_exists_cond probably the same as aggregate. Item_in_subselect::create_row_in_to_exists_cond not sure if it is possible to have window functions hare but should be checked. st_select_lex::check_unrestricted_recursive, pushdown_cond_for_derived It looks like here also should be checked. > /* > We cant change name of Item_field or Item_ref, because it will > prevent it's correct resolving, but we should save name of > _______________________________________________ > commits mailing list > commits(a)mariadb.org > https://lists.askmonty.org/cgi-bin/mailman/listinfo/commits

1 0

Re: [Maria-developers] [Commits] 88a80e92dc4: MDEV-9959: A serious MariaDB server performance bug
by Sergey Petrunia 27 Apr '19

27 Apr '19

Hi Varun, On Wed, Apr 24, 2019 at 01:31:38PM +0530, Varun wrote: > revision-id: 88a80e92dc444ce30718f3e08d3ab66fb02bcea4 (mariadb-10.3.10-284-g88a80e92dc4) > parent(s): 3032cd8e91f1e1ead8b6f941e75cd29e473e7eaa > author: Varun Gupta > committer: Varun Gupta > timestamp: 2019-04-24 13:31:24 +0530 > message: > > MDEV-9959: A serious MariaDB server performance bug > > Step #2: If any field in the select list of the derived tables is present in the group by list also , then we are again guaranteed > that ref access to the derived table would always produce one row per key. ... > diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc > index c52005e7683..aa645ac00ca 100644 > --- a/sql/sql_lex.cc > +++ b/sql/sql_lex.cc > @@ -7617,6 +7617,68 @@ Item *st_select_lex::build_cond_for_grouping_fields(THD *thd, Item *cond, > } > > > +/** > + Check if any any item in the group by list is also present in the select_list > + @retval true: All elements common between select and group by list > +*/ > + > +void st_select_lex::is_group_by_prefix(KEY *keyinfo) I think the name of the function is poor - "is_something()" hints at hints at boolean return value, and I would say it doesn't imply any side effects. Something like 'mark_derived_index_stats()' would be better, I think. > +{ > + uint key_parts= keyinfo->usable_key_parts; > + KEY_PART_INFO *key_part_info= keyinfo->key_part; > + bool found= FALSE; > + > + if (key_parts < group_list.elements) > + return; > + > + uint matched_fields=0, i, j; > + Item *item; > + > + for (i= 0; i < key_parts; key_part_info++, i++) > + { > + uint fld_idx= key_part_info->fieldnr - 1; > + item= join->fields_list.elem(fld_idx); > + for (ORDER *order= group_list.first; order; order= order->next) > + { > + Item *ord_item= order->item[0]->real_item(); > + Item_equal *item_equal= ord_item->get_item_equal(); > + > + if (item_equal) > + { > + Item_equal_fields_iterator it(*item_equal); > + Item *equal_item; > + while ((equal_item= it++)) > + { > + if (equal_item->eq(item, 0)) > + { > + matched_fields++; > + found= TRUE; > + break; > + } > + } > + } > + else > + { > + if (item->eq(ord_item, 0)) > + { > + matched_fields++; > + found= TRUE; > + } > + } > + if (found) > + break; > + } > + > + if (matched_fields == group_list.elements) > + { > + for (j=matched_fields - 1; j < key_parts; j++) > + keyinfo->rec_per_key[j]= 1; This doesn't always work. An example: create table t1 (a int, b int, c int, d int); insert into t1 select a,a,a,a from one_k; create table t2 as select * from t1; # Note the "a, a AS b" in the select list. This is intentional. explain select * from t2, (select a, a as b from t1 group by a,b) TBL where t2.a=TBL.a and t2.b=TBL.b; Here, the select output is not guaranteed to be unique (as it contains only one of two GROUP BY columns). But I see the execution to reac the above line, assigning keyinfo->rec_per_key[j]= 1. > + return; > + } > + found= FALSE; > + } > +} > + > int set_statement_var_if_exists(THD *thd, const char *var_name, > size_t var_name_length, ulonglong value) > { A general comment: do you think we should expose this in optimizer trace? At the moment, Derived-with-keys optimization doesn't print anything there (neither in MariaDB nor in MySQL). One can see the generated KEYUSEs when they are printed. But perhaps we could add a node that would print that a derived table was added, with this cardinality, and index with these keypart cardinalities. This would make debugging easier. Any thoughts? BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

Re: [Maria-developers] [Commits] 3032cd8e91f: step #1: if a derived table has SELECT DISTINCT, provide index statistics for it so that the join optimizer in the
by Sergey Petrunia 26 Apr '19

26 Apr '19

Hi Varun, Please mention the MDEV# in the commit comment. Otherwise, the patch is ok to push. On Mon, Apr 22, 2019 at 07:09:45PM +0530, Varun wrote: > revision-id: 3032cd8e91f1e1ead8b6f941e75cd29e473e7eaa (mariadb-10.3.10-283-g3032cd8e91f) > parent(s): f4019f5b3544a18f3ddf32df2c5214c3f8dabdce > author: Varun Gupta > committer: Varun Gupta > timestamp: 2019-04-22 18:19:25 +0530 > message: > > step #1: if a derived table has SELECT DISTINCT, provide index statistics for it so that the join optimizer in the > upper select knows that ref access to the table will produce one row. > > --- > mysql-test/main/cte_nonrecursive.result | 8 ++-- > mysql-test/main/derived.result | 54 ++++++++++++++++++++++ > mysql-test/main/derived.test | 30 ++++++++++++ > mysql-test/main/derived_view.result | 2 +- > mysql-test/main/subselect_extra.result | 2 +- > mysql-test/main/subselect_extra_no_semijoin.result | 2 +- > sql/sql_lex.h | 1 + > sql/sql_union.cc | 40 ++++++++++++++++ > sql/table.cc | 20 ++++++++ > 9 files changed, 152 insertions(+), 7 deletions(-) > > diff --git a/mysql-test/main/cte_nonrecursive.result b/mysql-test/main/cte_nonrecursive.result > index b846ec2d8ac..d80d34ecc7f 100644 > --- a/mysql-test/main/cte_nonrecursive.result > +++ b/mysql-test/main/cte_nonrecursive.result > @@ -244,7 +244,7 @@ with t as (select distinct a from t1 where b >= 'c') > select * from t as r1, t as r2 where r1.a=r2.a; > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY <derived2> ALL NULL NULL NULL NULL 8 Using where > -1 PRIMARY <derived3> ref key0 key0 5 r1.a 2 > +1 PRIMARY <derived3> ref key0 key0 5 r1.a 1 > 3 DERIVED t1 ALL NULL NULL NULL NULL 8 Using where; Using temporary > 2 DERIVED t1 ALL NULL NULL NULL NULL 8 Using where; Using temporary > explain > @@ -253,7 +253,7 @@ select * from (select distinct a from t1 where b >= 'c') as r1, > where r1.a=r2.a; > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY <derived2> ALL NULL NULL NULL NULL 8 Using where > -1 PRIMARY <derived3> ref key0 key0 5 r1.a 2 > +1 PRIMARY <derived3> ref key0 key0 5 r1.a 1 > 3 DERIVED t1 ALL NULL NULL NULL NULL 8 Using where; Using temporary > 2 DERIVED t1 ALL NULL NULL NULL NULL 8 Using where; Using temporary > # two references to t specified by a query > @@ -369,7 +369,7 @@ select c as a from t2 where c < 4) > select * from t2,t where t2.c=t.a; > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 4 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.c 2 > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.c 1 > 2 DERIVED t1 ALL NULL NULL NULL NULL 8 Using where > 3 UNION t2 ALL NULL NULL NULL NULL 4 Using where > NULL UNION RESULT <union2,3> ALL NULL NULL NULL NULL NULL > @@ -381,7 +381,7 @@ select c as a from t2 where c < 4) as t > where t2.c=t.a; > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 4 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.c 2 > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.c 1 > 2 DERIVED t1 ALL NULL NULL NULL NULL 8 Using where > 3 UNION t2 ALL NULL NULL NULL NULL 4 Using where > NULL UNION RESULT <union2,3> ALL NULL NULL NULL NULL NULL > diff --git a/mysql-test/main/derived.result b/mysql-test/main/derived.result > index f0d0289c1ce..857246d68b4 100644 > --- a/mysql-test/main/derived.result > +++ b/mysql-test/main/derived.result > @@ -1195,3 +1195,57 @@ drop table t1,t2,t3; > # > # End of 10.2 tests > # > +# > +# MDEV-9959: A serious MariaDB server performance bug > +# > +create table t1(a int); > +insert into t1 values (1),(2),(3),(4),(5),(6); > +create table t2(a int, b int,c int); > +insert into t2(a,b,c) values (1,1,2),(2,2,3),(3,1,4),(4,2,2),(5,1,1),(6,2,5); > +create table t3(a int, b int); > +insert into t3(a,b) values (1,1),(2,2),(2,1),(1,2),(5,1),(9,2); > +table "<derived2>" should have type=ref and rows=1 > +one select in derived table > +with distinct > +analyze select * from t1 , ((select distinct t2.a from t2 order by c))q where t1.a=q.a; > +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra > +1 PRIMARY t1 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t1.a 1 1.00 100.00 100.00 > +2 DERIVED t2 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 Using temporary; Using filesort > +analyze select * from t1 , ((select distinct t2.a, t2.b from t2 order by c))q where t1.a=q.a; > +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra > +1 PRIMARY t1 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t1.a 2 1.00 100.00 100.00 > +2 DERIVED t2 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 Using temporary; Using filesort > +# multiple selects in derived table > +# NO UNION ALL > +analyze select * from t1 , ( (select t2.a from t2 order by c) union (select t2.a from t2 order by c))q where t1.a=q.a; > +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra > +1 PRIMARY t1 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t1.a 1 1.00 100.00 100.00 > +2 DERIVED t2 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 > +3 UNION t2 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 > +NULL UNION RESULT <union2,3> ALL NULL NULL NULL NULL NULL 6.00 NULL NULL > +select * from t1 , ( (select t2.a from t2 order by c) union (select t2.a from t2 order by c))q where t1.a=q.a; > +a a > +1 1 > +2 2 > +3 3 > +4 4 > +5 5 > +6 6 > +# UNION ALL and EXCEPT > +analyze select * from t1 , ( (select t2.a from t2 order by c) union all (select t2.a from t2 order by c) except(select t3.a from t3 order by b))q where t1.a=q.a; > +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra > +1 PRIMARY t1 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t1.a 1 0.50 100.00 100.00 > +2 DERIVED t2 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 > +3 UNION t2 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 > +4 EXCEPT t3 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 > +NULL UNIT RESULT <unit2,3,4> ALL NULL NULL NULL NULL NULL 3.00 NULL NULL > +select * from t1 , ( (select t2.a from t2 order by c) union all (select t2.a from t2 order by c) except(select t3.a from t3 order by b))q where t1.a=q.a; > +a a > +3 3 > +4 4 > +6 6 > +drop table t1,t2,t3; > diff --git a/mysql-test/main/derived.test b/mysql-test/main/derived.test > index 6c51f23c51e..990f955450a 100644 > --- a/mysql-test/main/derived.test > +++ b/mysql-test/main/derived.test > @@ -1032,3 +1032,33 @@ drop table t1,t2,t3; > --echo # > --echo # End of 10.2 tests > --echo # > + > +--echo # > +--echo # MDEV-9959: A serious MariaDB server performance bug > +--echo # > + > +create table t1(a int); > +insert into t1 values (1),(2),(3),(4),(5),(6); > +create table t2(a int, b int,c int); > +insert into t2(a,b,c) values (1,1,2),(2,2,3),(3,1,4),(4,2,2),(5,1,1),(6,2,5); > +create table t3(a int, b int); > +insert into t3(a,b) values (1,1),(2,2),(2,1),(1,2),(5,1),(9,2); > + > +--echo table "<derived2>" should have type=ref and rows=1 > +--echo one select in derived table > + > +--echo with distinct > +analyze select * from t1 , ((select distinct t2.a from t2 order by c))q where t1.a=q.a; > +analyze select * from t1 , ((select distinct t2.a, t2.b from t2 order by c))q where t1.a=q.a; > + > +--echo # multiple selects in derived table > +--echo # NO UNION ALL > +analyze select * from t1 , ( (select t2.a from t2 order by c) union (select t2.a from t2 order by c))q where t1.a=q.a; > +select * from t1 , ( (select t2.a from t2 order by c) union (select t2.a from t2 order by c))q where t1.a=q.a; > + > +--echo # UNION ALL and EXCEPT > +analyze select * from t1 , ( (select t2.a from t2 order by c) union all (select t2.a from t2 order by c) except(select t3.a from t3 order by b))q where t1.a=q.a; > + > +select * from t1 , ( (select t2.a from t2 order by c) union all (select t2.a from t2 order by c) except(select t3.a from t3 order by b))q where t1.a=q.a; > + > +drop table t1,t2,t3; > diff --git a/mysql-test/main/derived_view.result b/mysql-test/main/derived_view.result > index 86dd73f5733..30831e75341 100644 > --- a/mysql-test/main/derived_view.result > +++ b/mysql-test/main/derived_view.result > @@ -1525,7 +1525,7 @@ EXPLAIN > SELECT a FROM t1 WHERE (a,b) IN (SELECT * FROM v2); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t1 ALL NULL NULL NULL NULL 2 Using where > -1 PRIMARY <derived3> ref key0 key0 10 test.t1.a,test.t1.b 2 FirstMatch(t1) > +1 PRIMARY <derived3> ref key0 key0 10 test.t1.a,test.t1.b 1 FirstMatch(t1) > 3 DERIVED t2 ALL NULL NULL NULL NULL 6 > 4 UNION t3 ALL NULL NULL NULL NULL 4 > NULL UNION RESULT <union3,4> ALL NULL NULL NULL NULL NULL > diff --git a/mysql-test/main/subselect_extra.result b/mysql-test/main/subselect_extra.result > index a3a0f1f9a15..dbcf00268c2 100644 > --- a/mysql-test/main/subselect_extra.result > +++ b/mysql-test/main/subselect_extra.result > @@ -409,7 +409,7 @@ EXPLAIN > SELECT a FROM t1 WHERE (a,b) IN (SELECT * FROM v2); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t1 ALL NULL NULL NULL NULL 2 Using where > -1 PRIMARY <derived3> ref key0 key0 10 test.t1.a,test.t1.b 2 FirstMatch(t1) > +1 PRIMARY <derived3> ref key0 key0 10 test.t1.a,test.t1.b 1 FirstMatch(t1) > 3 DERIVED t2 ALL NULL NULL NULL NULL 6 > 4 UNION t3 ALL NULL NULL NULL NULL 4 > NULL UNION RESULT <union3,4> ALL NULL NULL NULL NULL NULL > diff --git a/mysql-test/main/subselect_extra_no_semijoin.result b/mysql-test/main/subselect_extra_no_semijoin.result > index ec9ddb0452e..49a1431eb9b 100644 > --- a/mysql-test/main/subselect_extra_no_semijoin.result > +++ b/mysql-test/main/subselect_extra_no_semijoin.result > @@ -411,7 +411,7 @@ EXPLAIN > SELECT a FROM t1 WHERE (a,b) IN (SELECT * FROM v2); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t1 ALL NULL NULL NULL NULL 2 Using where > -2 DEPENDENT SUBQUERY <derived3> index_subquery key0 key0 10 func,func 2 Using where > +2 DEPENDENT SUBQUERY <derived3> index_subquery key0 key0 10 func,func 1 Using where > 3 DERIVED t2 ALL NULL NULL NULL NULL 6 > 4 UNION t3 ALL NULL NULL NULL NULL 4 > NULL UNION RESULT <union3,4> ALL NULL NULL NULL NULL NULL > diff --git a/sql/sql_lex.h b/sql/sql_lex.h > index 72ca4ac0b43..4eaec7d062b 100644 > --- a/sql/sql_lex.h > +++ b/sql/sql_lex.h > @@ -896,6 +896,7 @@ class st_select_lex_unit: public st_select_lex_node { > bool union_needs_tmp_table(); > > void set_unique_exclude(); > + bool check_distinct_in_union(); > > friend struct LEX; > friend int subselect_union_engine::exec(); > diff --git a/sql/sql_union.cc b/sql/sql_union.cc > index 87fbbebe4ba..3fb5552c77a 100644 > --- a/sql/sql_union.cc > +++ b/sql/sql_union.cc > @@ -2049,3 +2049,43 @@ void st_select_lex_unit::set_unique_exclude() > } > } > } > + > +/** > + @brief > + Check if the derived table is guaranteed to have distinct rows because of > + UNION operations used to populate it. > + > + @detail > + UNION operation removes duplicate rows from its output. That is, a query like > + > + select * from t1 UNION select * from t2 > + > + will not produce duplicate rows in its output, even if table t1 (and/or t2) > + contain duplicate rows. EXCEPT and INTERSECT operations also have this > + property. > + > + On the other hand, UNION ALL operation doesn't remove duplicates. (The SQL > + standard also defines EXCEPT ALL and INTERSECT ALL, but we don't support > + them). > + > + st_select_lex_unit computes its value left to right. That is, if there is > + a st_select_lex_unit object describing > + > + (select #1) OP1 (select #2) OP2 (select #3) > + > + then ((select #1) OP1 (select #2)) is computed first, and OP2 is computed > + second. > + > + How can one tell if st_select_lex_unit is guaranteed to have distinct > + output rows? This depends on whether the last operation was duplicate- > + removing or not: > + - UNION ALL is not duplicate-removing > + - all other operations are duplicate-removing > +*/ > + > +bool st_select_lex_unit::check_distinct_in_union() > +{ > + if (union_distinct && !union_distinct->next_select()) > + return true; > + return false; > +} > diff --git a/sql/table.cc b/sql/table.cc > index 80995abc1f9..c4494c9ae4b 100644 > --- a/sql/table.cc > +++ b/sql/table.cc > @@ -7269,6 +7269,26 @@ bool TABLE::add_tmp_key(uint key, uint key_parts, > key_part_info++; > } > > + /* > + For the case when there is a derived table that would give distinct rows, > + the index statistics are passed to the join optimizer to tell that a ref > + access to all the fields of the derived table will produce only one row. > + */ > + > + st_select_lex_unit* derived= pos_in_table_list ? > + pos_in_table_list->derived: NULL; > + if (derived) > + { > + st_select_lex* first= derived->first_select(); > + uint select_list_items= first->get_item_list()->elements; > + if (key_parts == select_list_items) > + { > + if ((!first->is_part_of_union() && (first->options & SELECT_DISTINCT)) || > + derived->check_distinct_in_union()) > + keyinfo->rec_per_key[key_parts - 1]= 1; > + } > + } > + > set_if_bigger(s->max_key_length, keyinfo->key_length); > s->keys++; > return FALSE; > _______________________________________________ > commits mailing list > commits(a)mariadb.org > https://lists.askmonty.org/cgi-bin/mailman/listinfo/commits -- BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

[Maria-developers] Big endian issue in libmariadb: test: main.mysql_client_test fails
by REIX, Tony 26 Apr '19

26 Apr '19

Hi, We are building MariaDB on AIX. The test: main.mysql_client_test fails (after time-out). This is due to little endian code being used though HAVE_BIGENDIAN is correctly set. This is due to: libmariadb/libmariadb/ma_dtoa.c : #if defined(WORDS_BIGENDIAN) || (defined(__FLOAT_WORD_ORDER) && \ (__FLOAT_WORD_ORDER == __BIG_ENDIAN)) #define word0(x) (x)->L[0] #define word1(x) (x)->L[1] #else #define word0(x) (x)->L[1] #define word1(x) (x)->L[0] #endif which results in the #else part (Little Endian) since none of the tested macros are defined. File libmariadb/libmariadb/ma_dtoa.c includes ma_global.h which has: #define HAVE_BIGENDIAN 1 but nothing defined about WORDS_BIGENDIAN . However, probably that some changes were made in the past in ma_global.h, since I see: #ifdef HAVE_BIGENDIAN #endif /* WORDS_BIGENDIAN */ 2 times in this file. I do not master MariaDB, so I do not have a clear idea about how to fix this. I see 3 solutions: a) libmariadb/CMakeLists.txt : Add: SET(HAVE_WORDS_BIGENDIAN TRUE ...) after : TEST_BIG_ENDIAN(HAVE_BIGENDIAN) if it is BigEndian. b) libmariadb/include/ma_global.h : Add: #ifdef HAVE_BIGENDIAN #define WORDS_BIGENDIAN 1 #endif c) libmariadb/libmariadb/ma_dtoa.c : Change: #if defined(WORDS_BIGENDIAN) || (defined(__FLOAT_WORD_ORDER) && \ (__FLOAT_WORD_ORDER == __BIG_ENDIAN)) by: #if defined(HAVE_BIGENDIAN) || defined(WORDS_BIGENDIAN) || (defined(__FLOAT_WORD_ORDER) && \ (__FLOAT_WORD_ORDER == __BIG_ENDIAN)) Please let me know which seems appropriate. Regards, Tony ATOS / Bull SAS ATOS Expert IBM Coop Architect & Technical Leader Office : +33 (0) 4 76 29 72 67 1 rue de Provence - 38432 Échirolles - France www.atos.net<https://mail.ad.bull.net/owa/redir.aspx?C=PvphmPvCZkGrAgHVnWGsdMcDKgzl_dEIs…>

1 0

Re: [Maria-developers] [Commits] 647c22692ad: MDEV-17588 replicate-do filters cause errors when creating filtered-out tables on master with syntax unsupported on slave
by andrei.elkin＠pp.inet.fi 25 Apr '19

25 Apr '19

Sachin, howdy. > revision-id: 647c22692ad1fe466eb14b749520cc520807d4fa (mariadb-10.1.38-114-g647c22692ad) > parent(s): 6c5e4c9bc0d9ac30f7ec7ee334630bacb58687ba > author: Sachin > committer: Sachin > timestamp: 2019-04-23 17:58:03 +0530 > message: > > MDEV-17588 replicate-do filters cause errors when creating filtered-out tables on master with syntax unsupported on slave > > Prototype Commit. In next commit could you please brief about the problem and attempted solution. > > Delay throwing of error in parsing stage untill we know the result from rpl > filter. I think you were on the right track implementing this deferred error/warning issuing. Thanks for attempting it! There's a new THD::"flag" for that purpose. I suggest we place it inside struct LEX_MASTER_INFO to set pretty similarly to how the patch does. Secondly, let blocks in sql_yacc.yy that call error/warning functions that gets deferred (because of the slave thread) be invoked (if they will be) in mysql_execute_command() in verbatim. That means to turn the blocks into functions and call them at once or later. Your patch presents two such blocks, I point to them below. For each block I suggest we introduce a flag. The block flag is set when the block gets visited (but not executed 'cos of slave). The "flag" therefore is going to be an array (or enum?). > > --- > mysql-test/suite/rpl/r/rpl_mdev_17588.result | 26 ++++++++++++++++++++++ > mysql-test/suite/rpl/t/rpl_mdev_17588-slave.opt | 1 + > mysql-test/suite/rpl/t/rpl_mdev_17588.test | 29 +++++++++++++++++++++++++ > sql/sql_class.h | 1 + > sql/sql_parse.cc | 7 ++++++ > sql/sql_yacc.yy | 12 +++++++--- > 6 files changed, 73 insertions(+), 3 deletions(-) > > diff --git a/mysql-test/suite/rpl/r/rpl_mdev_17588.result b/mysql-test/suite/rpl/r/rpl_mdev_17588.result > new file mode 100644 > index 00000000000..976efde2bdd > --- /dev/null > +++ b/mysql-test/suite/rpl/r/rpl_mdev_17588.result > @@ -0,0 +1,26 @@ > +include/master-slave.inc > +[connection master] > +set sql_log_bin= 0; > +install soname 'ha_tokudb'; > +set sql_log_bin= 1; > +create table t1 (a int) engine=TokuDB; > +create table t2 (a int); > +create table t3 (a int) engine=TokuDB; > +include/wait_for_slave_sql_error.inc [errno=1286] > +show create table t1; > +ERROR 42S02: Table 'test.t1' doesn't exist > +show create table t2; > +Table Create Table > +t2 CREATE TABLE `t2` ( > + `a` int(11) DEFAULT NULL > +) ENGINE=MyISAM DEFAULT CHARSET=latin1 > +show create table t3; > +ERROR 42S02: Table 'test.t3' doesn't exist > +SET GLOBAL SQL_SLAVE_SKIP_COUNTER=1; > +START SLAVE; > +drop table t1, t2, t3; > +set sql_log_bin= 0; > +uninstall soname 'ha_tokudb'; > +set sql_log_bin= 1; > +CALL mtr.add_suppression('Slave: Unknown storage engine .* Error_code: 1286'); > +include/rpl_end.inc > diff --git a/mysql-test/suite/rpl/t/rpl_mdev_17588-slave.opt b/mysql-test/suite/rpl/t/rpl_mdev_17588-slave.opt > new file mode 100644 > index 00000000000..19497afd22a > --- /dev/null > +++ b/mysql-test/suite/rpl/t/rpl_mdev_17588-slave.opt > @@ -0,0 +1 @@ > +--replicate-do-table=test.t2 --replicate-do-table=test.t3 --sql-mode='NO_ENGINE_SUBSTITUTION' > diff --git a/mysql-test/suite/rpl/t/rpl_mdev_17588.test b/mysql-test/suite/rpl/t/rpl_mdev_17588.test > new file mode 100644 > index 00000000000..c636077e4d1 > --- /dev/null > +++ b/mysql-test/suite/rpl/t/rpl_mdev_17588.test > @@ -0,0 +1,29 @@ > +--source include/master-slave.inc > + > +--connection master > +set sql_log_bin= 0; > +install soname 'ha_tokudb'; > +set sql_log_bin= 1; > + > +create table t1 (a int) engine=TokuDB; > +create table t2 (a int); > +create table t3 (a int) engine=TokuDB; > + > +--connection slave > +let $slave_sql_errno= 1286; > +source include/wait_for_slave_sql_error.inc; > +--error ER_NO_SUCH_TABLE > +show create table t1; > +show create table t2; > +--error ER_NO_SUCH_TABLE > +show create table t3; > +SET GLOBAL SQL_SLAVE_SKIP_COUNTER=1; > +START SLAVE; > +--connection master > +drop table t1, t2, t3; > +set sql_log_bin= 0; > +uninstall soname 'ha_tokudb'; > +set sql_log_bin= 1; > +--sync_slave_with_master > +CALL mtr.add_suppression('Slave: Unknown storage engine .* Error_code: 1286'); > +--source include/rpl_end.inc > diff --git a/sql/sql_class.h b/sql/sql_class.h > index 6640e02147a..4361700ee6c 100644 > --- a/sql/sql_class.h > +++ b/sql/sql_class.h > @@ -2802,6 +2802,7 @@ class THD :public Statement, > uint8 password; /* 0, 1 or 2 */ > uint8 failed_com_change_user; > bool slave_thread; > + char *rpl_unavailable_storage_plugin= NULL; > bool extra_port; /* If extra connection */ > > bool no_errors; > diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc > index 6649c60f827..5f8c833c76f 100644 > --- a/sql/sql_parse.cc > +++ b/sql/sql_parse.cc > @@ -2656,6 +2656,13 @@ mysql_execute_command(THD *thd) > /* we warn the slave SQL thread */ > my_message(ER_SLAVE_IGNORED_TABLE, ER_THD(thd, ER_SLAVE_IGNORED_TABLE), > MYF(0)); > + thd->rpl_unavailable_storage_plugin= NULL; > + DBUG_RETURN(0); > + } > + if (thd->rpl_unavailable_storage_plugin) > + { > + my_error(ER_UNKNOWN_STORAGE_ENGINE, MYF(0), thd->rpl_unavailable_storage_plugin); > + thd->rpl_unavailable_storage_plugin= NULL; Here the check would go for each thd->lex.mi.flag[i] when set call "flag_func[i]"; I index flag[] but I don't insist on implementing it like that. Here is the function's body { if (thd->variables.sql_mode & MODE_NO_ENGINE_SUBSTITUTION) my_yyabort_error((ER_UNKNOWN_STORAGE_ENGINE, MYF(0), $1.str)); $$= 0; push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, ER_UNKNOWN_STORAGE_ENGINE, ER_THD(thd, ER_UNKNOWN_STORAGE_ENGINE), $1.str); } and a flag function also might need some parameters like $1.str in above. > DBUG_RETURN(0); > } > /* > diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy > index 5111f0690ab..0756028c34a 100644 > --- a/sql/sql_yacc.yy > +++ b/sql/sql_yacc.yy > @@ -2476,7 +2476,7 @@ create: > LEX *lex= thd->lex; > lex->current_select= &lex->select_lex; > if ((lex->create_info.used_fields & HA_CREATE_USED_ENGINE) && > - !lex->create_info.db_type) > + !lex->create_info.db_type && !thd->rpl_unavailable_storage_plugin) > { > lex->create_info.use_default_db_type(thd); This is the 2nd block which depends on the 1st one. if ((lex->create_info.used_fields & HA_CREATE_USED_ENGINE) && !lex->create_info.db_type) { lex->create_info.use_default_db_type(thd); if (!thd->slave_thread || !lex->mi.flag[0]) print_warn_using_other_handler(...) } where print_warn_using_other_handler() is { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, ER_WARN_USING_OTHER_HANDLER, ER_THD(thd, ER_WARN_USING_OTHER_HANDLER), hton_name(lex->create_info.db_type)->str, $5->table.str); } To call it deferredly we seem need two args to memorize (reserve fields too) in Lex->mi. I hope my suggestions make sense and feasible. Cheers, Andrei > push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, > @@ -5791,9 +5791,15 @@ storage_engines: > else > { > if (thd->variables.sql_mode & MODE_NO_ENGINE_SUBSTITUTION) > - my_yyabort_error((ER_UNKNOWN_STORAGE_ENGINE, MYF(0), $1.str)); > + { > + if (!thd->slave_thread) > + my_yyabort_error((ER_UNKNOWN_STORAGE_ENGINE, MYF(0), $1.str)); > + else > + thd->rpl_unavailable_storage_plugin= $1.str; > + } > $$= 0; > - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, > + if (!thd->rpl_unavailable_storage_plugin) > + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, > ER_UNKNOWN_STORAGE_ENGINE, > ER_THD(thd, ER_UNKNOWN_STORAGE_ENGINE), > $1.str); > _______________________________________________ > commits mailing list > commits(a)mariadb.org > https://lists.askmonty.org/cgi-bin/mailman/listinfo/commits

1 0

Re: [Maria-developers] [Commits] 8ae7a5eb037: MDEV-17260: Memory leaks in mysqlbinlog
by andrei.elkin＠pp.inet.fi 24 Apr '19

24 Apr '19

Sujatha, hello. The patch looks good, I only have a suggestion to the test for better readability, within the quoted patch below. Thanks for a good piece of work! Cheers, Andrei >commit 8ae7a5eb03725441393c353e4e203f80b666c035 >Author: Sujatha Sivakumar <sujatha.sivakumar(a)mariadb.com> >Date: Tue Apr 23 19:47:59 2019 +0530 > > MDEV-17260: Memory leaks in mysqlbinlog > > Problem: > ======== > The mysqlbinlog tool is leaking memory, causing failures in various tests when > compiling and testing with AddressSanitizer or LeakSanitizer like this: > > cmake -DCMAKE_BUILD_TYPE=Debug -DWITH_ASAN:BOOL=ON /path/to/source > make -j$(nproc) > cd mysql-test > ASAN_OPTIONS=abort_on_error=1 ./mtr --parallel=auto > > Analysis: > ========= > Two types of leaks were observed during above execution. > > 1) Leak in Log_event::read_log_event(char const*, unsigned int, char const**, > Format_description_log_event const*, char) > File: sql/log_event.cc:2150 > > For all row based replication events the memory which is allocated during > read_log_event is not freed after the event is processed. The event specific > memory has to be retained only when flashback option is enabled with > mysqlbinlog tool. In this case all the events are retained till the end > statement is received and they are processed in reverse order and they are > destroyed. But in the existing code all events are retained irrespective of > flashback mode. Hence the memory leaks are observed. > > 2) read_remote_annotate_event(unsigned char*, unsigned long, char const**) > File: client/mysqlbinlog.cc:194 > > In general the Annotate event is not printed immediately because all > subsequent rbr-events can be filtered away. Instead it will be printed > together with the first not filtered away Table map or the last rbr will be > processed. While reading remote annotate events memory is allocated for event > buffer and event's temp_buf is made to point to the allocated buffer as shown > below. The TRUE flag is used for doing proper cleanup using free_temp_buf(). > i.e at the time of deletion of annotate event its destructor takes care of > clearing the temp_buf. > > /* > Ensure the event->temp_buf is pointing to the allocated buffer. > (TRUE = free temp_buf on the event deletion) > */ > event->register_temp_buf((char*)event_buf, TRUE); > > But existing code does the following when it receives a remote annotate_event. > > if (remote_opt) > ev->temp_buf= 0; > > That is code immediately sets temp_buf=0, because of which free_temp_buf() > call will return empty handed as it has lost the reference to the allocated > temporary buffer. This results in memory leak > > Fix: > ==== > 1) If not in flashback mode, destroy the memory for events once they are > processed. > > 2) Remove the ev->temp_buf=0 code for remote option. Let the proper cleanup to > be done as part of free_temp_buf(). > >diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc >index cfc05cbf794..3dad4fef67b 100644 >--- a/client/mysqlbinlog.cc >+++ b/client/mysqlbinlog.cc >@@ -227,8 +227,7 @@ void print_annotate_event(PRINT_EVENT_INFO *print_event_info) > if (annotate_event) > { > annotate_event->print(result_file, print_event_info); >- delete annotate_event; // the event should not be printed more than once >- annotate_event= 0; >+ free_annotate_event(); > } > } > >@@ -1465,7 +1464,7 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, > if (print_row_event(print_event_info, ev, e->get_table_id(), > e->get_flags(Rows_log_event::STMT_END_F))) > goto err; >- if (!is_stmt_end) >+ if (opt_flashback && !is_stmt_end) > destroy_evt= FALSE; > break; > } >@@ -1478,7 +1477,7 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, > if (print_row_event(print_event_info, ev, e->get_table_id(), > e->get_flags(Old_rows_log_event::STMT_END_F))) > goto err; >- if (!is_stmt_end) >+ if (opt_flashback && !is_stmt_end) > destroy_evt= FALSE; > break; > } >@@ -1539,8 +1538,6 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, > } > } > >- if (remote_opt) >- ev->temp_buf= 0; > if (destroy_evt) /* destroy it later if not set (ignored table map) */ > delete ev; > } >diff --git a/mysql-test/suite/binlog/r/flashback.result b/mysql-test/suite/binlog/r/flashback.result >index c96eaebe838..047c032b5df 100644 >--- a/mysql-test/suite/binlog/r/flashback.result >+++ b/mysql-test/suite/binlog/r/flashback.result >@@ -674,6 +674,28 @@ world.city 563256876 > DROP TABLE test.test; > DROP TABLE world.city; > DROP DATABASE world; >+# < CASE 7 > >+# Test Case for MDEV-17260 >+# >+RESET MASTER; >+CREATE TABLE t1 ( f INT PRIMARY KEY ) ENGINE=innodb; >+INSERT INTO t1 VALUES (1),(2),(3),(4),(5),(6); >+# 6- Rows must be present >+SELECT COUNT(*) FROM t1; >+COUNT(*) >+6 >+FLUSH LOGS; >+DELETE FROM t1; >+FLUSH LOGS; >+# 0- Rows must be present >+SELECT COUNT(*) FROM t1; >+COUNT(*) >+0 >+# 6- Rows must be present upon restoring from flashback >+SELECT COUNT(*) FROM t1; >+COUNT(*) >+6 >+DROP TABLE t1; > SET binlog_format=statement; > Warnings: > Warning 1105 MariaDB Galera and flashback do not support binlog format: STATEMENT >diff --git a/mysql-test/suite/binlog/t/flashback.test b/mysql-test/suite/binlog/t/flashback.test >index 3fc8c44c60c..fb80669ca51 100644 >--- a/mysql-test/suite/binlog/t/flashback.test >+++ b/mysql-test/suite/binlog/t/flashback.test >@@ -335,8 +335,29 @@ DROP TABLE test.test; > DROP TABLE world.city; > DROP DATABASE world; > >-## Clear >+--echo # < CASE 7 > >+--echo # Test Case for MDEV-17260 >+--echo # >+RESET MASTER; >+ >+CREATE TABLE t1 ( f INT PRIMARY KEY ) ENGINE=innodb; >+INSERT INTO t1 VALUES (1),(2),(3),(4),(5),(6); >+--echo # 6- Rows must be present >+SELECT COUNT(*) FROM t1; SELECT COUNT(*) as six FROM t1; would read better though it's far from being the optimal automation. (The optimal one clearly must also count 'six' or whatever number itself, as well as bail out with an error when count(*) won't match the expected # - sure not for the current patch). >+FLUSH LOGS; >+DELETE FROM t1; >+FLUSH LOGS; >+--echo # 0- Rows must be present >+SELECT COUNT(*) FROM t1; >+--exec $MYSQL_BINLOG -vv -B --read-from-remote-server --user=root --host=127.0.0.1 --port=$MASTER_MYPORT master-bin.000002> $MYSQLTEST_VARDIR/tmp/mysqlbinlog_row_flashback_7.sql >+--exec $MYSQL -e "SET binlog_format= ROW; source $MYSQLTEST_VARDIR/tmp/mysqlbinlog_row_flashback_7.sql;" >+ >+--echo # 6- Rows must be present upon restoring from flashback >+SELECT COUNT(*) FROM t1; > >+DROP TABLE t1; >+ >+## Clear > SET binlog_format=statement; > --error ER_FLASHBACK_NOT_SUPPORTED > SET GLOBAL binlog_format=statement; >

1 0

Re: [Maria-developers] 70c5ab5658d: Move XID_STATE::rm_error to XID_cache_element
by Sergei Golubchik 24 Apr '19

24 Apr '19

Hi, Sergey! On Apr 24, Sergey Vojtovich wrote: > revision-id: 70c5ab5658d (mariadb-10.4.4-29-g70c5ab5658d) > parent(s): c5b1799435b > author: Sergey Vojtovich <svoj(a)mariadb.org> > committer: Sergey Vojtovich <svoj(a)mariadb.org> > timestamp: 2019-04-20 14:45:16 +0400 > message: > > Move XID_STATE::rm_error to XID_cache_element It's somewhat confusing. What if it's an internal 2PC and some engine's rollback returns an error? The error won't be stored in xid_state.rm_error anymore. Basically, you're changing the semantics of rm_error from "an error that happened during rollback" to "an error that happened during rollback or an external XA transaction". Mention this change explicitly in the commit comment, please. > Also removed redundant xid reset from THD::init_for_queries(). Must've > been done already either by THD::transaction constructor or by > THD::cleanup(). > > Part of MDEV-7974 - backport fix for mysql bug#12161 (XA and binlog) > > diff --git a/sql/sql_class.h b/sql/sql_class.h > index c68628be65d..e7584a0d825 100644 > --- a/sql/sql_class.h > +++ b/sql/sql_class.h > @@ -2606,13 +2606,8 @@ class THD: public THD_count, /* this must be first */ > DBUG_ENTER("thd::cleanup"); rename to "THD::st_transaction::cleanup" please > changed_tables= 0; > savepoints= 0; > - /* > - If rm_error is raised, it means that this piece of a distributed > - transaction has failed and must be rolled back. But the user must > - rollback it explicitly, so don't start a new distributed XA until > - then. > - */ > - if (!xid_state.rm_error) > + /* xid_cache_delete() resets xid of explicitly started XA transaction */ > + if (!xid_state.is_explicit_XA()) > xid_state.xid.null(); > free_root(&mem_root,MYF(MY_KEEP_PREALLOC)); > DBUG_VOID_RETURN; Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

1 0

[Maria-developers] C API prepared statements parameters for IN (x, x, x, ...)
by Simon Walter 24 Apr '19

24 Apr '19

I want to add support for this or sponsor it - if it is really true that this is not possible with the current c API. I have been told to do: SELECT x FROM x WHERE id IN (?, ?); Where the content of the parenthesis are not dynamic. Rather than going at it brute force on my own, which is my tendency, I want to hear what other developers think and if someone, perhaps a core dev, wants to, I would like to sponsor it. Sorry for the short email. It's late here in Japan. I have been thinking about this for months. Until now I have been using all sorts of work-arounds, but I think that dealing with the core issue will be a better use of my time/resources. More experienced developers may state reasons why this is not possible. Though as you may be able to tell, I can see how it could work. Of course I may be not seeing how everyone else is doing it. From what I can tell (lots code grepping, asking on mailing lists), noone in doing it. In my opinion, that is a shame. Because an, AFAICT, an IN clause means one disk read. The application I am working on make extensive use of IN clauses for reading many records in one call. Some may call this an micro-optimization. If that is really so, please show me that thousands of individual calls to a prepared statement like: SELECT x FROM x WHERE id = x; is as efficient as one call to: SELECT x FROM x WHERE id IN (x, x, x, ...); Background: I am using APR Util and have asked a few people about this and it boils down to the underlying drivers not supporting it. I would ultimately like to code a solution for pgsql as well. Mariadb is where I would like to start however. Then add support for it in APR. Suggesting I use sprintf to create dynamic queries means that I need to write my own sprintf, because as you know, va_arg etc. An array is actually dynamic - not va_arg - reading from file etc. Please feel free to tell me I am stupid, suggest efficient alternatives, tell me that it is actually supported (point me to the docs), etc. SSDs and such are great. Though, I still think it is a shame to be that lazy. No, I do not think the world will overheat because of our lazy programming. I have to throw that in there with the current political climate. No pun intended. ;) Thanks, Simon PS I do not work for any stock trading companies. The project I am working on is opensource. I just want to engineer something very nice.

1 1

[Maria-developers] mysql-proxy to maxscale migration
by Witold Filipczyk 24 Apr '19

24 Apr '19

Hi, I have simple lua script: cdb = require("cdb") function read_auth() local db = cdb.open("/etc/mysql/proxy/data.cdb") local value = db:get(proxy.connection.client.username) db:close() if value == nil then proxy.response.type = proxy.MYSQLD_PACKET_ERR proxy.response.errmsg = "Access denied" return proxy.PROXY_SEND_RESULT end end Format of data: username1 1 username2 1 and so on. Unfortunately after upgrade from mariadb-10.1 to mariadb-10.3 on buster mysql-proxy is defunct. Cannot get "mysql prompt". I tried to replace mysql-proxy by maxscale-2.3.6, but without success. Failed to load users from server 'server1': Lost connection to MySQL server during query Aborted connection 8578832 to db: 'mysql' user: 'debian-sys-maint' host: 'localhost' (Got an error writing communication packets) Note that there is a few hundreds thousands users in mysql.user, but in data.cdb only a few hundreds. Could you point me to the right place, where to find info how to fix mysql-proxy, or configure maxscale, or some alternative to both? Witold Filipczyk

2 1

[Maria-developers] Please review a patch for MDEV-19284 and MDEV-19285 (instant ALTER)
by Alexander Barkov 23 Apr '19

23 Apr '19

Hi Marko and Eugene, please review a patch fixing: - MDEV-19284 INSTANT ALTER with ucs2-to-utf16 conversion produces bad data - MDEV-19285 INSTANT ALTER from ascii_general_ci to latin1_general_ci produces currupt data Thanks!

3 8

[Maria-developers] SELECT INTO OUTFILE + LOAD DATA for a mixture of character and binary data
by Alexander Barkov 22 Apr '19

22 Apr '19

Hi Kenneth, I just closed this issue as "not a bug": MDEV-13361 Regression: LOAD DATA INFILE utf8 enconding issues Can you please document that when doing "SELECT INTO OUTFILE" followed by "LOAD DATA INFILE" for a mixture of text and binary data, one should use "CHARACTER SET binary" in both statements. This is needed to avoid escape sequences to be interpreted in a wrong way at "LOAD DATA" type. Thanks.

1 0

Re: [Maria-developers] [Commits] 167a96b5157: MDEV-9959: A serious MariaDB server performance bug
by Sergey Petrunia 20 Apr '19

20 Apr '19

On Tue, Mar 27, 2018 at 04:48:10PM +0530, Varun wrote: > revision-id: 167a96b5157049408a6ad4bca7abcd376af93fb5 (mariadb-10.3.0-644-g167a96b5157) > parent(s): 4359c6b4806605c78987e50cab3a6b42016b7603 > author: Varun Gupta > committer: Varun Gupta > timestamp: 2018-03-27 16:45:46 +0530 > message: > > MDEV-9959: A serious MariaDB server performance bug > > step#1: if a derived table has SELECT DISTINCT, provide index statistics for it so that the join optimizer in the > upper select knows that ref access to the table will produce one row. > > Added handling for multiple selects in the derived table > > --- > mysql-test/r/cte_nonrecursive.result | 8 +-- > mysql-test/r/cte_recursive.result | 6 +- > mysql-test/r/derived_cond_pushdown.result | 82 ++++++++++++------------- > mysql-test/r/derived_view.result | 2 +- > mysql-test/r/join_cache.result | 6 +- > mysql-test/r/mdev9959.result | 46 ++++++++++++++ > mysql-test/r/subselect_extra.result | 2 +- > mysql-test/r/subselect_extra_no_semijoin.result | 2 +- > mysql-test/t/mdev9959.test | 25 ++++++++ > sql/sql_lex.h | 2 + > sql/sql_union.cc | 60 ++++++++++++++++++ > sql/sql_yacc.yy | 3 + > sql/table.cc | 19 ++++++ > 13 files changed, 209 insertions(+), 54 deletions(-) > > diff --git a/mysql-test/r/derived_cond_pushdown.result b/mysql-test/r/derived_cond_pushdown.result > index 32d3c88cc8d..3723e25a494 100644 > --- a/mysql-test/r/derived_cond_pushdown.result > +++ b/mysql-test/r/derived_cond_pushdown.result > @@ -5187,7 +5187,7 @@ explain select * from v2_union as v,t2 where > ((v.a=6) or (v.a=8)) and (v.c>200) and (v.a=t2.a); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 6 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 Using where As agreed on the call: need to check what is the cause of this. Here, the temp table has distinct rows, that is, {a,b,c} are distinct. But ref access only uses the first component, where does rows=1 come from? (if this is how best_access_path computes an estimate for prefix when it only has the estimate for the full key ... fine) > 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort > 3 UNION t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort > 4 UNION t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort > @@ -5213,7 +5213,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["a"], > "ref": ["test.t2.a"], > - "rows": 6, > + "rows": 1, > "filtered": 100, > "attached_condition": "v.c > 200", > "materialized": { > @@ -5358,7 +5358,7 @@ a b c a b c d > explain select * from v3_union as v,t2 where (v.a=t2.a) and (v.c>6); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 4 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 Using where > 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where > 3 UNION t1 ALL NULL NULL NULL NULL 20 Using where > NULL UNION RESULT <union2,3> ALL NULL NULL NULL NULL NULL > @@ -5382,7 +5382,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["a"], > "ref": ["test.t2.a"], > - "rows": 4, > + "rows": 1, > "filtered": 100, > "attached_condition": "v.c > 6", > "materialized": { > @@ -5476,7 +5476,7 @@ a b c a b c d > explain select * from v3_union as v,t2 where (v.a=t2.a) and ((t2.a>1) or (v.b<20)); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 4 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 Using where > 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where > 3 UNION t1 ALL NULL NULL NULL NULL 20 Using where > NULL UNION RESULT <union2,3> ALL NULL NULL NULL NULL NULL > @@ -5500,7 +5500,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["a"], > "ref": ["test.t2.a"], > - "rows": 4, > + "rows": 1, > "filtered": 100, > "attached_condition": "t2.a > 1 or v.b < 20", > "materialized": { > @@ -5561,7 +5561,7 @@ explain select * from v3_union as v,t2 where > (v.a=t2.a) and ((v.b=19) or (v.b=21)) and ((v.c<3) or (v.c>600)); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 4 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 Using where > 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where > 3 UNION t1 ALL NULL NULL NULL NULL 20 Using where > NULL UNION RESULT <union2,3> ALL NULL NULL NULL NULL NULL > @@ -5586,7 +5586,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["a"], > "ref": ["test.t2.a"], > - "rows": 4, > + "rows": 1, > "filtered": 100, > "attached_condition": "(v.b = 19 or v.b = 21) and (v.c < 3 or v.c > 600)", > "materialized": { > @@ -5645,7 +5645,7 @@ a b c a b c d > explain select * from v4_union as v,t2 where (v.a=t2.a) and (v.b<20); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 4 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 Using where > 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort > 3 UNION t1 ALL NULL NULL NULL NULL 20 Using where > NULL UNION RESULT <union2,3> ALL NULL NULL NULL NULL NULL > @@ -5669,7 +5669,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["a"], > "ref": ["test.t2.a"], > - "rows": 4, > + "rows": 1, > "filtered": 100, > "attached_condition": "v.b < 20", > "materialized": { > @@ -5752,7 +5752,7 @@ explain select * from v4_union as v,t2 where > (v.a=t2.a) and ((t2.a<3) or (v.b<40)) and (v.c>500); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 4 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 Using where > 2 DERIVED t1 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort > 3 UNION t1 ALL NULL NULL NULL NULL 20 Using where > NULL UNION RESULT <union2,3> ALL NULL NULL NULL NULL NULL > @@ -5777,7 +5777,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["a"], > "ref": ["test.t2.a"], > - "rows": 4, > + "rows": 1, > "filtered": 100, > "attached_condition": "(t2.a < 3 or v.b < 40) and v.c > 500", > "materialized": { > @@ -7952,7 +7952,7 @@ a b c a b c > explain select * from v1,t2 where (v1.a=t2.a) and (v1.a<5); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 2 > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 > 2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > 3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > NULL EXCEPT RESULT <except2,3> ALL NULL NULL NULL NULL NULL > @@ -7976,7 +7976,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["a"], > "ref": ["test.t2.a"], > - "rows": 2, > + "rows": 1, > "filtered": 100, > "materialized": { > "query_block": { > @@ -8226,7 +8226,7 @@ a b c a b c > explain select * from v1,t2 where (v1.a=t2.a) and (v1.c>500); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 2 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 Using where > 2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > 3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > NULL EXCEPT RESULT <except2,3> ALL NULL NULL NULL NULL NULL > @@ -8250,7 +8250,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["a"], > "ref": ["test.t2.a"], > - "rows": 2, > + "rows": 1, > "filtered": 100, > "attached_condition": "v1.c > 500", > "materialized": { > @@ -8317,7 +8317,7 @@ a b c a b c > explain select * from v1,t2 where (v1.a=t2.a) and (v1.a<5) and (v1.c>500); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 2 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 Using where > 2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > 3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > NULL EXCEPT RESULT <except2,3> ALL NULL NULL NULL NULL NULL > @@ -8341,7 +8341,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["a"], > "ref": ["test.t2.a"], > - "rows": 2, > + "rows": 1, > "filtered": 100, > "attached_condition": "v1.c > 500", > "materialized": { > @@ -8411,7 +8411,7 @@ a b c a b c > explain select * from v1,t2 where (v1.a=t2.a) and ((v1.b>27) or (v1.b<19)); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 2 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 Using where > 2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > 3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > NULL EXCEPT RESULT <except2,3> ALL NULL NULL NULL NULL NULL > @@ -8435,7 +8435,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["a"], > "ref": ["test.t2.a"], > - "rows": 2, > + "rows": 1, > "filtered": 100, > "attached_condition": "v1.b > 27 or v1.b < 19", > "materialized": { > @@ -8508,7 +8508,7 @@ explain select * from v1,t2 where > (v1.a=t2.a) and ((v1.c<400) or (v1.c>800)); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 2 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 Using where > 2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > 3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > NULL EXCEPT RESULT <except2,3> ALL NULL NULL NULL NULL NULL > @@ -8533,7 +8533,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["a"], > "ref": ["test.t2.a"], > - "rows": 2, > + "rows": 1, > "filtered": 100, > "attached_condition": "v1.c < 400 or v1.c > 800", > "materialized": { > @@ -8762,7 +8762,7 @@ where > ((d1.a>4) and (d1.c>500))); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.b 2 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.b 1 Using where > 2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > 3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > NULL EXCEPT RESULT <except2,3> ALL NULL NULL NULL NULL NULL > @@ -8796,7 +8796,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["b"], > "ref": ["test.t2.b"], > - "rows": 2, > + "rows": 1, > "filtered": 100, > "attached_condition": "t2.c = 988 and t2.b > 13 or d1.a > 4 and d1.c > 500", > "materialized": { > @@ -8872,7 +8872,7 @@ a b c a b c > explain select * from v1,t2 where (v1.a=t2.a) and (v1.a>5) and (v1.c>200); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 3 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 Using where > 2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > 3 UNION <derived4> ALL NULL NULL NULL NULL 18 Using where > 4 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > @@ -8899,7 +8899,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["a"], > "ref": ["test.t2.a"], > - "rows": 3, > + "rows": 1, > "filtered": 100, > "attached_condition": "v1.c > 200", > "materialized": { > @@ -9138,7 +9138,7 @@ a b c a b c > explain select * from v1,t2 where (v1.a=t2.a) and (v1.a>5) and (v1.c>200); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 3 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 Using where > 2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > 3 UNION t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > 4 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > @@ -9163,7 +9163,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["a"], > "ref": ["test.t2.a"], > - "rows": 3, > + "rows": 1, > "filtered": 100, > "attached_condition": "v1.c > 200", > "materialized": { > @@ -9260,7 +9260,7 @@ a b c a b c > explain select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<200); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 3 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 Using where > 2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > 3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > 4 UNION t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > @@ -9285,7 +9285,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["a"], > "ref": ["test.t2.a"], > - "rows": 3, > + "rows": 1, > "filtered": 100, > "attached_condition": "v1.c < 200", > "materialized": { > @@ -9498,7 +9498,7 @@ a b c a b c > explain select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<130); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 2 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 Using where > 2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > 3 EXCEPT <derived4> ALL NULL NULL NULL NULL 18 Using where > 4 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > @@ -9525,7 +9525,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["a"], > "ref": ["test.t2.a"], > - "rows": 2, > + "rows": 1, > "filtered": 100, > "attached_condition": "v1.c < 130", > "materialized": { > @@ -9643,7 +9643,7 @@ a b c a b c > explain select * from v1,t2 where (v1.a=t2.a) and (v1.a>4) and (v1.c<130); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 3 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 Using where > 2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > 3 EXCEPT <derived4> ALL NULL NULL NULL NULL 18 Using where > 4 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > @@ -9671,7 +9671,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["a"], > "ref": ["test.t2.a"], > - "rows": 3, > + "rows": 1, > "filtered": 100, > "attached_condition": "v1.c < 130", > "materialized": { > @@ -10143,7 +10143,7 @@ a b c a b c > explain select * from v1,t2 where (v1.a=t2.a) and (v1.a<2) and (v1.b<30) and (v1.c>450); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 2 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 Using where > 2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > 3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > NULL EXCEPT RESULT <except2,3> ALL NULL NULL NULL NULL NULL > @@ -10167,7 +10167,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["a"], > "ref": ["test.t2.a"], > - "rows": 2, > + "rows": 1, > "filtered": 100, > "attached_condition": "v1.b < 30 and v1.c > 450", > "materialized": { > @@ -10243,7 +10243,7 @@ a b c a b c > explain select * from v1,t2 where (v1.a=t2.a) and ((v1.a<2) or (v1.a<5)) and (v1.c>450); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 2 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 Using where > 2 DERIVED t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > 3 EXCEPT t1 ALL NULL NULL NULL NULL 18 Using where; Using temporary; Using filesort > NULL EXCEPT RESULT <except2,3> ALL NULL NULL NULL NULL NULL > @@ -10267,7 +10267,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["a"], > "ref": ["test.t2.a"], > - "rows": 2, > + "rows": 1, > "filtered": 100, > "attached_condition": "v1.c > 450", > "materialized": { > @@ -10636,7 +10636,7 @@ a b c a b c > explain select * from v1,t2 where (v1.b=t2.b) and (v1.a<3); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 9 Using where > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.b 2 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.b 1 Using where > 2 DERIVED t3 range i1 i1 5 NULL 1 Using index condition > 3 UNION t3 ALL NULL NULL NULL NULL 20 Using where; Using temporary; Using filesort > NULL UNION RESULT <union2,3> ALL NULL NULL NULL NULL NULL > @@ -10660,7 +10660,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["b"], > "ref": ["test.t2.b"], > - "rows": 2, > + "rows": 1, > "filtered": 100, > "attached_condition": "v1.a < 3", > "materialized": { > @@ -12333,7 +12333,7 @@ EXPLAIN > "key_length": "5", > "used_key_parts": ["b"], > "ref": ["test.t1.a"], > - "rows": 2, > + "rows": 1, > "filtered": 100, > "materialized": { > "query_block": { > diff --git a/mysql-test/r/derived_view.result b/mysql-test/r/derived_view.result > index 85e56ff176e..3a0fde7b053 100644 > --- a/mysql-test/r/derived_view.result > +++ b/mysql-test/r/derived_view.result > @@ -1525,7 +1525,7 @@ EXPLAIN > SELECT a FROM t1 WHERE (a,b) IN (SELECT * FROM v2); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t1 ALL NULL NULL NULL NULL 2 Using where > -1 PRIMARY <derived3> ref key0 key0 10 test.t1.a,test.t1.b 2 FirstMatch(t1) > +1 PRIMARY <derived3> ref key0 key0 10 test.t1.a,test.t1.b 1 FirstMatch(t1) Interesting.. here, we potentially could infer that this semi-join can be converted to an inner join (as the subquery may only have one match). But I think this is outside of the scope of this MDEV. > 3 DERIVED t2 ALL NULL NULL NULL NULL 6 > 4 UNION t3 ALL NULL NULL NULL NULL 4 > NULL UNION RESULT <union3,4> ALL NULL NULL NULL NULL NULL > diff --git a/mysql-test/r/join_cache.result b/mysql-test/r/join_cache.result > index eea397402ad..cc185b640bc 100644 > --- a/mysql-test/r/join_cache.result > +++ b/mysql-test/r/join_cache.result > @@ -5197,7 +5197,7 @@ SELECT * FROM (SELECT DISTINCT * FROM t1) t > WHERE t.a IN (SELECT t2.a FROM t2); > id select_type table type possible_keys key key_len ref rows Extra > 1 PRIMARY t2 ALL NULL NULL NULL NULL 4 Using where; Start temporary > -1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 2 End temporary > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 End temporary > 2 DERIVED t1 ALL NULL NULL NULL NULL 3 Using temporary > SELECT * FROM (SELECT DISTINCT * FROM t1) t > WHERE t.a IN (SELECT t2.a FROM t2); > @@ -5208,8 +5208,8 @@ EXPLAIN > SELECT * FROM (SELECT DISTINCT * FROM t1) t > WHERE t.a IN (SELECT t2.a FROM t2); > id select_type table type possible_keys key key_len ref rows Extra > -1 PRIMARY <derived2> ALL NULL NULL NULL NULL 3 > -1 PRIMARY t2 ALL NULL NULL NULL NULL 4 Using where; Start temporary; End temporary; Using join buffer (flat, BNL join) > +1 PRIMARY t2 ALL NULL NULL NULL NULL 4 Using where; Start temporary > +1 PRIMARY <derived2> ref key0 key0 5 test.t2.a 1 End temporary > 2 DERIVED t1 ALL NULL NULL NULL NULL 3 Using temporary > SELECT * FROM (SELECT DISTINCT * FROM t1) t > WHERE t.a IN (SELECT t2.a FROM t2); > diff --git a/mysql-test/r/mdev9959.result b/mysql-test/r/mdev9959.result > new file mode 100644 > index 00000000000..049e0350cca > --- /dev/null > +++ b/mysql-test/r/mdev9959.result > @@ -0,0 +1,46 @@ > +create table t1(a int); > +insert into t1 values (1),(2),(3),(4),(5),(6); > +create table t2(a int, b int,c int); > +insert into t2(a,b,c) values (1,1,2),(2,2,3),(3,1,4),(4,2,2),(5,1,1),(6,2,5); > +create table t3(a int, b int); > +insert into t3(a,b) values (1,1),(2,2),(2,1),(1,2),(5,1),(9,2); > +table "<derived2>" should have type=ref and rows=1 > +one select in derived table > +with distinct > +analyze select * from t1 , ((select distinct t2.a from t2 order by c))q where t1.a=q.a; > +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra > +1 PRIMARY t1 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t1.a 1 1.00 100.00 100.00 > +2 DERIVED t2 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 Using temporary; Using filesort > +# multiple selects in derived table > +# NO UNION ALL > +analyze select * from t1 , ( (select t2.a,t2.b from t2 order by c) union (select t2.a,t2.b from t2 order by c))q where t1.a=q.a; > +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra > +1 PRIMARY t1 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t1.a 1 1.00 100.00 100.00 > +2 DERIVED t2 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 > +3 UNION t2 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 > +NULL UNION RESULT <union2,3> ALL NULL NULL NULL NULL NULL 6.00 NULL NULL > +select * from t1 , ( (select t2.a from t2 order by c) union (select t2.a from t2 order by c))q where t1.a=q.a; > +a a > +1 1 > +2 2 > +3 3 > +4 4 > +5 5 > +6 6 > +# UNION ALL and EXCEPT > +analyze select * from t1 , ( (select t2.a,t2.b from t2 order by c) union all (select t2.a,t2.b from t2 order by c) except(select t3.a, t3.b from t3 order by b))q where t1.a=q.a; > +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra > +1 PRIMARY t1 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t1.a 1 0.50 100.00 100.00 > +2 DERIVED t2 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 > +3 UNION t2 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 > +4 EXCEPT t3 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 > +NULL UNIT RESULT <unit2,3,4> ALL NULL NULL NULL NULL NULL 3.00 NULL NULL > +select * from t1 , ( (select t2.a,t2.b from t2 order by c) union all (select t2.a,t2.b from t2 order by c) except(select t3.a, t3.b from t3 order by b))q where t1.a=q.a; > +a a b > +3 3 1 > +4 4 2 > +6 6 2 > +drop table t1,t2,t3; > diff --git a/mysql-test/t/mdev9959.test b/mysql-test/t/mdev9959.test > new file mode 100644 > index 00000000000..36f364261c9 > --- /dev/null > +++ b/mysql-test/t/mdev9959.test > @@ -0,0 +1,25 @@ > +create table t1(a int); > +insert into t1 values (1),(2),(3),(4),(5),(6); > +create table t2(a int, b int,c int); > +insert into t2(a,b,c) values (1,1,2),(2,2,3),(3,1,4),(4,2,2),(5,1,1),(6,2,5); > +create table t3(a int, b int); > +insert into t3(a,b) values (1,1),(2,2),(2,1),(1,2),(5,1),(9,2); > + > +--echo table "<derived2>" should have type=ref and rows=1 > +--echo one select in derived table > + > +--echo with distinct > +analyze select * from t1 , ((select distinct t2.a from t2 order by c))q where t1.a=q.a; > + > +--echo # multiple selects in derived table > +--echo # NO UNION ALL > +analyze select * from t1 , ( (select t2.a,t2.b from t2 order by c) union (select t2.a,t2.b from t2 order by c))q where t1.a=q.a; > +select * from t1 , ( (select t2.a from t2 order by c) union (select t2.a from t2 order by c))q where t1.a=q.a; > + > +--echo # UNION ALL and EXCEPT > +analyze select * from t1 , ( (select t2.a,t2.b from t2 order by c) union all (select t2.a,t2.b from t2 order by c) except(select t3.a, t3.b from t3 order by b))q where t1.a=q.a; > + > +select * from t1 , ( (select t2.a,t2.b from t2 order by c) union all (select t2.a,t2.b from t2 order by c) except(select t3.a, t3.b from t3 order by b))q where t1.a=q.a; > + > +drop table t1,t2,t3; > + > diff --git a/sql/sql_lex.h b/sql/sql_lex.h > index f0241a32acf..16d5e55d251 100644 > --- a/sql/sql_lex.h > +++ b/sql/sql_lex.h > @@ -761,6 +761,7 @@ class st_select_lex_unit: public st_select_lex_node { > Procedure *last_procedure; /* Pointer to procedure, if such exists */ > > bool columns_are_renamed; > + bool union_all; /* TRUE if there is a UNION ALL operation */ This is not initialized properly. The testcase: create table t10 (a int , b int); insert into t10 values (1,1),(1,1); create table t11 (a int , b int); create table t12 (a int , b int); insert into t11 values (2,2),(2,2); create table t13 (a int , b int); insert into t12 values (3,3),(3,3); create table ten(a int); insert into ten values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); explain select * from ten A, (select * from t10 union select * from t11 union select * from t12) T where T.a=A.a; +------+--------------+--------------+------+---------------+------+---------+----------+------+-------------+ | id | select_type | table | type | possible_keys | key | key_len | ref | rows | Extra | +------+--------------+--------------+------+---------------+------+---------+----------+------+-------------+ | 1 | PRIMARY | A | ALL | NULL | NULL | NULL | NULL | 10 | Using where | | 1 | PRIMARY | <derived2> | ref | key0 | key0 | 5 | test.A.a | 1 | | | 2 | DERIVED | t10 | ALL | NULL | NULL | NULL | NULL | 2 | | | 3 | UNION | t11 | ALL | NULL | NULL | NULL | NULL | 2 | | | 4 | UNION | t12 | ALL | NULL | NULL | NULL | NULL | 2 | | | NULL | UNION RESULT | <union2,3,4> | ALL | NULL | NULL | NULL | NULL | NULL | | +------+--------------+--------------+------+---------------+------+---------+----------+------+-------------+ (gdb) wher #0 st_select_lex_unit::check_distinct_in_union (this=0x7fff700149e0) at /home/psergey/dev-git/10.3-cp/sql/sql_union.cc:2100 #1 0x0000555555d2a693 in TABLE::add_tmp_key (this=0x7fff70062908, key=0, key_parts=1, next_field_no=0x555555c6bf47 <get_next_field_for_derived_key(uchar*)>, arg=0x7fffd98e9230 "X\207\006p\377\177", unique=false) at /home/psergey/dev-git/10.3-cp/sql/table.cc:7270 #2 0x0000555555c6c214 in generate_derived_keys_for_table (keyuse=0x7fff700687b0, count=1, keys=1) at /home/psergey/dev-git/10.3-cp/sql/sql_select.cc:11091 #3 0x0000555555c6c42a in generate_derived_keys (keyuse_array=0x7fff70018968) at /home/psergey/dev-git/10.3-cp/sql/sql_select.cc:11166 #4 0x0000555555c60267 in sort_and_filter_keyuse (thd=0x7fff70000d60, keyuse=0x7fff70018968, skip_unprefixed_keyparts=true) at /home/psergey/dev-git/10.3-cp/sql/sql_select.cc:6403 #5 0x0000555555c5a910 in make_join_statistics (join=0x7fff70018678, tables_list=..., keyuse_array=0x7fff70018968) at /home/psergey/dev-git/10.3-cp/sql/sql_select.cc:4622 #6 0x0000555555c513d4 in JOIN::optimize_inner (this=0x7fff70018678) at /home/psergey/dev-git/10.3-cp/sql/sql_select.cc:1888 #7 0x0000555555c4fa73 in JOIN::optimize (this=0x7fff70018678) at /home/psergey/dev-git/10.3-cp/sql/sql_select.cc:1451 #8 0x0000555555c59638 in mysql_select (thd=0x7fff70000d60, tables=0x7fff70013f68, wild_num=1, fields=..., conds=0x7fff70017d98, og_num=0, order=0x0, group=0x0, having=0x0, proc_param=0x0, select_options=2147748612, result=0x7fff70017ff0, unit=0x7fff70004c28, select_lex=0x7fff70005398) at /home/psergey/dev-git/10.3-cp/sql/sql_select.cc:4225 #9 0x0000555555c91d6f in mysql_explain_union (thd=0x7fff70000d60, unit=0x7fff70004c28, result=0x7fff70017ff0) at /home/psergey/dev-git/10.3-cp/sql/sql_select.cc:25866 #10 0x0000555555c15da6 in execute_sqlcom_select (thd=0x7fff70000d60, all_tables=0x7fff70013f68) at /home/psergey/dev-git/10.3-cp/sql/sql_parse.cc:6487 #11 0x0000555555c0c7a5 in mysql_execute_command (thd=0x7fff70000d60) at /home/psergey/dev-git/10.3-cp/sql/sql_parse.cc:3821 #12 0x0000555555c1a0fd in mysql_parse (thd=0x7fff70000d60, rawbuf=0x7fff70013ca8 "explain select * from ten A, (select * from t10 union select * from t11 union select * from t12) T where T.a=A.a", length=112, parser_state=0x7fffd98ea5d0, is_com_multi=false, is_next_command=false) at /home/psergey/dev-git/10.3-cp/sql/sql_parse.cc:8091 (gdb) p union_all $36 = 165 > > void init_query(); > st_select_lex* outer_select(); > @@ -800,6 +801,7 @@ class st_select_lex_unit: public st_select_lex_node { > bool union_needs_tmp_table(); > > void set_unique_exclude(); > + bool check_distinct_in_union(); can this be made 'const'? if yes, please make it > > friend struct LEX; > friend int subselect_union_engine::exec(); > diff --git a/sql/sql_union.cc b/sql/sql_union.cc > index 857c9a117f5..6dac0cb0458 100644 > --- a/sql/sql_union.cc > +++ b/sql/sql_union.cc > @@ -1960,3 +1960,63 @@ void st_select_lex_unit::set_unique_exclude() > } > } > } > + > +/* > + Check if the selects in the derived table can give distinct rows irrespective > + of the data given for the tables. > + > + for example: > + select * from > + ((select t1.a from t1) op (select t2.a from t2) op (select t3.a from t3)); > + the op here being UNION/INTERSECT/EXCEPT > + > + so this function would check if the derived table like the case above > + can give distinct rows or not irrespective of the data in the tables. I think the above is difficult to read. Readability is a matter of opinion, but I would suggest another variant: /* @brief Check if the derived table is guaranteed to have distinct rows because of UNION operations used to populate it. @detail UNION operation removes duplicate rows from its output. That is, a query like select * from t1 UNION select * from t2 will not produce duplicate rows in its output, even if table t1 (and/or t2) contain duplicate rows. EXCEPT and INTERSECT operations also have this property. On the other hand, UNION ALL operation doesn't remove duplicates. (The SQL standard also defines EXCEPT ALL and INTERSECT ALL, but we don't support them). st_select_lex_unit computes its value left to right. That is, if there is a st_select_lex_unit object describing (select #1) OP1 (select #2) OP2 (select #3) then ((select #1) OP1 (select #2)) is computed first, and OP2 is computed second. How can one tell if st_select_lex_unit is guaranteed to have distinct output rows? This depends on whether the last operation was duplicate- removing or not: - UNION ALL is not duplicate-removing - all other operations are duplicate-removing */ Is the above correct? > + > + So what the function is handling: > + - If there is no UNION ALL, we are guarenteed distinct rows. > + > + Example > + ((select t1.a from t1) UNION (select t2.a from t2); > + > + - If there is UNION ALL, we can still guarantee distinct rows if > + the last operation of the selects does not involve UNION ALL or > + INTERSECT is not there . > + > + Example: > + ((select t1.a from t1) UNION ALL (select t2.a from t2) UNION/EXCEPT > + (select t3.a from t3)): guarantees distinct rows, UNION and EXCEPT > + have the same precedence. > + > + Example: > + ((select t1.a from t1) UNION ALL (select t2.a from t2) INTERSECT > + (select t3.a from t3)): does not guarantee distinct rows because > + INTERSECT has higher precedence than UNION so we would evaluate > + the INTERSECT part first and then do UNION ALL, so there we can > + end up with duplicates, so distinct rows are not guaranteed. > + > + @retval false Distinct rows are not guaranteed > + @retval true Distinct rows are guanranteed irrespective of the data > + in the tables > + > +*/ > + > +bool st_select_lex_unit::check_distinct_in_union() > +{ > + bool is_intersect_present=FALSE; > + st_select_lex* first= first_select(); > + for(st_select_lex *sl=first; sl ; sl=sl->next_select()) > + is_intersect_present|= (sl->linkage == INTERSECT_TYPE); > + > + if (!union_all) > + return true; > + else > + { > + if (union_distinct) > + { > + if (!is_intersect_present && !union_distinct->next_select()) > + return true; > + } I'm unable to undestand what kind of check we are making here. According to my variant of the comment above, we should just check whether the last operation (operation made with the output of the last select) was duplicate-removing or not. We seem to be doing something more complex? > + } > + return false; > +} > diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy > index 35ec2d29d21..112873e8362 100644 > --- a/sql/sql_yacc.yy > +++ b/sql/sql_yacc.yy > @@ -571,7 +571,10 @@ bool LEX::add_select_to_union_list(bool is_union_distinct, > current_select; > } > else > + { > DBUG_ASSERT(type == UNION_TYPE); > + current_select->master_unit()->union_all= true; > + } > return FALSE; > } > > diff --git a/sql/table.cc b/sql/table.cc > index 4f90d429ce5..d2938a29ea6 100644 > --- a/sql/table.cc > +++ b/sql/table.cc > @@ -7113,6 +7113,25 @@ bool TABLE::add_tmp_key(uint key, uint key_parts, > if (!keyinfo->rec_per_key) > return TRUE; > bzero(keyinfo->rec_per_key, sizeof(ulong)*key_parts); > + /* > + For the case when there is a derived table that would give distinct rows, > + the index statistics are passed to the join optimizer to tell that > + a ref access to the derived table will produce only one row. > + */ > + > + st_select_lex_unit* derived= pos_in_table_list ? pos_in_table_list->derived: NULL; > + if (derived) > + { > + /* > + This handles the case when we have a single select in the derived table > + */ Does it anymore? Now it is handling the UNION case, too. > + st_select_lex* first= derived->first_select(); > + if ((first && !first->is_part_of_union() && > + first->options & SELECT_DISTINCT) || please put brackets around the condition on the line above as some too-smart compilers now complain about a possible typo. > + derived->check_distinct_in_union()) > + keyinfo->rec_per_key[key_parts-1]=1; > + } > + > keyinfo->read_stats= NULL; > keyinfo->collected_stats= NULL; > BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

Re: [Maria-developers] [Commits] 167a96b5157: MDEV-9959: A serious MariaDB server performance bug
by Sergey Petrunia 18 Apr '19

18 Apr '19

On Tue, Mar 27, 2018 at 04:48:10PM +0530, Varun wrote: > revision-id: 167a96b5157049408a6ad4bca7abcd376af93fb5 (mariadb-10.3.0-644-g167a96b5157) > parent(s): 4359c6b4806605c78987e50cab3a6b42016b7603 > author: Varun Gupta > committer: Varun Gupta > timestamp: 2018-03-27 16:45:46 +0530 > message: > > MDEV-9959: A serious MariaDB server performance bug > > step#1: if a derived table has SELECT DISTINCT, provide index statistics for it so that the join optimizer in the > upper select knows that ref access to the table will produce one row. > > Added handling for multiple selects in the derived table ... > diff --git a/mysql-test/r/mdev9959.result b/mysql-test/r/mdev9959.result > new file mode 100644 > index 00000000000..049e0350cca > --- /dev/null > +++ b/mysql-test/r/mdev9959.result > @@ -0,0 +1,46 @@ > +create table t1(a int); > +insert into t1 values (1),(2),(3),(4),(5),(6); > +create table t2(a int, b int,c int); > +insert into t2(a,b,c) values (1,1,2),(2,2,3),(3,1,4),(4,2,2),(5,1,1),(6,2,5); > +create table t3(a int, b int); > +insert into t3(a,b) values (1,1),(2,2),(2,1),(1,2),(5,1),(9,2); > +table "<derived2>" should have type=ref and rows=1 > +one select in derived table > +with distinct > +analyze select * from t1 , ((select distinct t2.a from t2 order by c))q where t1.a=q.a; > +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra > +1 PRIMARY t1 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t1.a 1 1.00 100.00 100.00 > +2 DERIVED t2 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 Using temporary; Using filesort > +# multiple selects in derived table > +# NO UNION ALL > +analyze select * from t1 , ( (select t2.a,t2.b from t2 order by c) union (select t2.a,t2.b from t2 order by c))q where t1.a=q.a; > +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra > +1 PRIMARY t1 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 Using where > +1 PRIMARY <derived2> ref key0 key0 5 test.t1.a 1 1.00 100.00 100.00 > +2 DERIVED t2 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 > +3 UNION t2 ALL NULL NULL NULL NULL 6 6.00 100.00 100.00 > +NULL UNION RESULT <union2,3> ALL NULL NULL NULL NULL NULL 6.00 NULL NULL How would you explain the above? The derived table is a union of two selects, each of which produces 6 rows. Neither of the selects have DISTINCT attribute. Why does the optimization fire and set ref=1 in this case? (if both parts of UNION had distinct, this would make sense. But if neither does, I don't see any logic) BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

[Maria-developers] MariaDB 10.4 on AIX. Issue with: HAVE_POOL_OF_THREADS
by REIX, Tony 18 Apr '19

18 Apr '19

Hi, In my environment (AIX & MariaDB v10.4.3 or 10.4.4), I have an issue with HAVE_POOL_OF_THREADS. When building, I have the following trace and error (full line of g++ compilation at end of email) : [ 44%] Linking CXX executable explain_filename-t /opt/freeware/bin/cmake -E cmake_link_script CMakeFiles/explain_filename-t.dir/link.txt --verbose=1 /opt/freeware/bin/g++ ........... -o explain_filename-t .... ld: 0711-317 ERROR: Undefined symbol: .TP_pool_generic::TP_pool_generic() ld: 0711-317 ERROR: Undefined symbol: .TP_pool_generic::init() This seems to be due to the following lines of the file: sql/CMakeLists.txt : IF (CMAKE_SYSTEM_NAME MATCHES "Linux" OR CMAKE_SYSTEM_NAME MATCHES "Windows" OR CMAKE_SYSTEM_NAME MATCHES "SunOS" OR HAVE_KQUEUE) ADD_DEFINITIONS(-DHAVE_POOL_OF_THREADS) IF(WIN32) SET(SQL_SOURCE ${SQL_SOURCE} threadpool_win.cc) ENDIF() SET(SQL_SOURCE ${SQL_SOURCE} threadpool_generic.cc) ENDIF() In case the Operating System is not Linux, Windows, or SunOS, then there is NO -DHAVE_POOL_OF_THREADS and the file: threadpool_generic.cc is NOT compiled. However, the C++ code of MariaDB still looks at least for 2 routines of threadpool_generic.cc : TP_pool_generic::TP_pool_generic() TP_pool_generic::init() but these routines are not available since threadpool_generic.cc was not compiled. So, in my opinion, there are some #ifdef HAVE_POOL_OF_THREADS lines that are missing in common threadpool code that would prevent to use threadpool generic routines. However, I do not master this code. Why not implement on AIX the generic pool of thread and thus add AIX to the above IF( ) test of sql/CMakeLists.txt ?? The reason if that, out of the 4 possibilities listed in sql/threadpool_generic.cc : epoll, KQUEUE, port, and IOCP, none of them are available on AIX with the EXACT required behavior. We have to implement something miming epoll(), we think. Before fixing this performance improvement, we'd like to be able to build & test MariaDB for all its other features. sql/threadpool_generic.cc : #ifdef __linux__ #include <sys/epoll.h> typedef struct epoll_event native_event; #elif defined(HAVE_KQUEUE) #include <sys/event.h> typedef struct kevent native_event; #elif defined (__sun) #include <port.h> typedef port_event_t native_event; #elif defined (HAVE_IOCP) typedef OVERLAPPED_ENTRY native_event; #else #error threadpool is not available on this platform <<---- AIX #endif Regards, Tony /opt/freeware/bin/g++ -O2 -g -L/opt/freeware/lib64 -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -maix64 -Wl,-bbigtoc -fPIC -O2 -g -L/opt/freeware/lib64 -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -maix64 -Wl,-bbigtoc -fPIC -pie -fPIC -fno-rtti -O3 -g -static-libgcc -fno-omit-frame-pointer -fno-strict-aliasing -Wno-uninitialized -D_FORTIFY_SOURCE=2 -DDBUG_OFF -Wl,-blibpath:/opt/freeware/lib:/opt/freeware/lib64:/usr/lib:/lib -lpgport CMakeFiles/explain_filename-t.dir/explain_filename-t.cc.o -o explain_filename-t -Wl,-bnoipath -Wl,-blibpath:/opt/freeware/lib64:/opt/freeware/lib/gcc/powerpc-ibm-aix7.2.0.0/8.2.0/ppc64:/opt/freeware/lib/ppc64:/opt/freeware/lib/gcc/powerpc-ibm-aix7.2.0.0/8.2.0:/opt/freeware/lib:/usr/lib:/lib -lpthreads ../../sql/libsql.a ../mytap/libmytap.a ../../storage/archive/libarchive.a ../../storage/blackhole/libblackhole.a ../../storage/csv/libcsv.a ../../storage/federatedx/libfederatedx.a ../../storage/heap/libheap.a ../../storage/innobase/libinnobase.a -llz4 ../../storage/maria/libaria.a ../../storage/myisam/libmyisam.a ../../storage/myisammrg/libmyisammrg.a ../../storage/sequence/libsequence.a ../../plugin/feedback/libfeedback.a ../../plugin/user_variables/libuser_variables.a ../../plugin/userstat/libuserstat.a ../../sql/libpartition.a ../../sql/libsql_sequence.a ../../mysys/libmysys.a ../../mysys_ssl/libmysys_ssl.a ../../dbug/libdbug.a ../../mysys/libmysys.a ../../mysys_ssl/libmysys_ssl.a ../../dbug/libdbug.a /usr/lib64/libz.so -lm ../../extra/yassl/libyassl.a ../../extra/yassl/taocrypt/libtaocrypt.a ../../strings/libstrings.a ../../vio/libvio.a ../../pcre/libpcre.a ../../wsrep-lib/src/libwsrep-lib.a -lpthread -ldl ../../wsrep-lib/wsrep-API/libwsrep_api_v26.a -lpthreads

2 5

Re: [Maria-developers] [JIRA] (MDEV-17260) Memory leaks in mysqlbinlog
by andrei.elkin＠pp.inet.fi 17 Apr '19

17 Apr '19

Sujatha, hello. The patches look good though I have a concern in .. > [ https://jira.mariadb.org/browse/MDEV-17260?page=com.atlassian.jira.plugin.s… ] > > Sujatha Sivakumar commented on MDEV-17260: > ------------------------------------------ > > In the above report there are three different kinds of leaks were observed. > > In 5.5: > ===== > CURRENT_TEST: rpl.rpl_row_mysqlbinlog > ================================================================ > ==8988==ERROR: LeakSanitizer: detected memory leaks > Direct leak of 112 byte(s) in 1 object(s) allocated from: > #0 0x4eff87 in __interceptor_malloc (/dev/shm/5.5/client/mysqlbinlog+0x4eff87) > #1 0x60eaab in my_malloc /mariadb/5.5/mysys/my_malloc.c:41:10 > #2 0x5300dd in Log_event::read_log_event(char const*, unsigned int, char const**, Format_description_log_event const*, char) /mariadb/5.5/sql/log_event.cc:1568:12 > #3 0x564a9c in dump_remote_log_entries(st_print_event_info*, char const*) /mariadb/5.5/client/mysqlbinlog.cc:1978:17 > #4 0x536408 in dump_log_entries(char const*) /mariadb/5.5/client/mysqlbinlog.cc:1794:21 > #5 0x535e4b in main /mariadb/5.5/client/mysqlbinlog.cc:2505:18 > #6 0x7f71ea0a1b16 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x22b16) > > This issue exists only in 5.5. It is specific to fake rotate events while reading binary log from a remote server. > This issues is already fixed in MariaDB 10.0.23 and higher versions as part of > commit c3018b0ff4fb02c029787d03867adf0530607bab > > 10.2 and above: > ============= > 1) Leak in Log_event::read_log_event(char const*, unsigned int, char const**, > Format_description_log_event const*, char) > File: sql/log_event.cc:2150 .. in 2). > 2) read_remote_annotate_event(unsigned char*, unsigned long, char const**) > File: client/mysqlbinlog.cc:194 > What would you think could be the reason of the being deleted two-liner: @@ -1539,10 +1539,11 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, } } - if (remote_opt) - ev->temp_buf= 0; I see there were done as a part of the flashback feature. Do we have the flashback and remote combination in any test? Cheers, Andrei > These were caused because of flashback feature and its related bug fixes. > > To reproduce the failures use: > > ASAN_OPTIONS=abort_on_error=1 ./mtr --parallel=auto binlog.binlog_row_mysqlbinlog_verbose binlog_encryption.binlog_row_annotate binlog.binlog_row_mysqlbinlog_options binlog.binlog_row_annotate encryption.tempfiles binlog.binlog_mysqlbinlog_row_myisam binlog.binlog_mysqlbinlog_row binlog.binlog_mysqlbinlog_base64 binlog_encryption.rpl_skip_replication binlog.binlog_mysqlbinlog_row_innodb binlog.flashback > >> Memory leaks in mysqlbinlog >> --------------------------- >> >> Key: MDEV-17260 >> URL: https://jira.mariadb.org/browse/MDEV-17260 >> Project: MariaDB Server >> Issue Type: Bug >> Components: Scripts & Clients >> Affects Versions: 10.0, 10.1, 10.2, 10.3, 10.4, 5.5.62 >> Reporter: Marko Mäkelä >> Assignee: Sujatha Sivakumar >> Priority: Major >> Labels: ASAN, Memory_leak, valgrind >> Fix For: 10.1, 10.2, 10.3, 10.4 >> >> Attachments: mdev-17260-wip.patch >> >> Time Spent: 5h >> Remaining Estimate: 0d >> >> The {{mysqlbinlog}} tool is leaking memory, causing failures in various tests when compiling and testing with AddressSanitizer or LeakSanitizer like this: >> {code:sh} >> cmake -DCMAKE_BUILD_TYPE=Debug -DWITH_ASAN:BOOL=ON /path/to/source >> make -j$(nproc) >> cd mysql-test >> ASAN_OPTIONS=abort_on_error=1 ./mtr --parallel=auto >> {code} >> {noformat:title=5.5 e07118946a82af60c7cc3804c321d3fd9a49f128} >> CURRENT_TEST: rpl.rpl_row_mysqlbinlog >> ================================================================= >> ==8988==ERROR: LeakSanitizer: detected memory leaks >> Direct leak of 112 byte(s) in 1 object(s) allocated from: >> #0 0x4eff87 in __interceptor_malloc (/dev/shm/5.5/client/mysqlbinlog+0x4eff87) >> #1 0x60eaab in my_malloc /mariadb/5.5/mysys/my_malloc.c:41:10 >> #2 0x5300dd in Log_event::read_log_event(char const*, unsigned int, char const**, Format_description_log_event const*, char) /mariadb/5.5/sql/log_event.cc:1568:12 >> #3 0x564a9c in dump_remote_log_entries(st_print_event_info*, char const*) /mariadb/5.5/client/mysqlbinlog.cc:1978:17 >> #4 0x536408 in dump_log_entries(char const*) /mariadb/5.5/client/mysqlbinlog.cc:1794:21 >> #5 0x535e4b in main /mariadb/5.5/client/mysqlbinlog.cc:2505:18 >> #6 0x7f71ea0a1b16 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x22b16) >> {noformat} > > > > -- > This message was sent by Atlassian Jira > (v7.13.1#713001)

1 0

[Maria-developers] MariaDB 10.4 on AIX. Issue with: sql/wsrep_mysqld.cc : ip_len
by REIX, Tony 16 Apr '19

16 Apr '19

Hi, In my environment (AIX & MariaDB v10.4.3 or 10.4.4), the following line of sql/wsrep_mysqld.cc : size_t const ip_len= wsrep_host_len(node_addr.c_str(), node_addr.size()); is preprocessed (gcc -E) as: size_t const ip_ff.ip_vhltl.ip_x.ip_xlen = wsrep_host_len(node_addr.c_str(), node_addr.size()); where "ip_len" becomes: ip_ff.ip_vhltl.ip_x.ip_xlen , which is wrong and breaks. This is due to the following macro in AIX /usr/include/netinet/ip.h : #define ip_len ip_ff.ip_flen The attached patch (replace ip_len by ip_len_mdb) is an easy work-around for fixing the issue. However, it may not be the perfect appropriate change. Since I do not master MariaDB, I'd like someone to propose a fully correct fix. Thanks/Regards, Tony Reix

1 0

[Maria-developers] MariaDB 10.4 on AIX. Issue with: #define SI_KERNEL in include/my_pthread.h
by REIX, Tony 16 Apr '19

16 Apr '19

Hi, In my environment (AIX & MariaDB v10.4.3 or 10.4.4), I have: include/config.h: #define HAVE_SIGWAITINFO 1 and thus in the file : include/my_pthread.h : static inline int my_sigwait(sigset_t *set, int *sig, int *code) { #ifdef HAVE_SIGWAITINFO siginfo_t siginfo; *sig= sigwaitinfo(set, &siginfo); *code= siginfo.si_code; return *sig < 0 ? errno : 0; #else #define SI_KERNEL 128 *code= 0; return sigwait(set, sig); #endif } the SI_KERNEL macro is NOT defined. Then, I get the following error at compilation: mariadb-10.4.3/sql/mysqld.cc:3272:36: error: 'SI_KERNEL' was not declared in this scope if (!abort_loop && origin != SI_KERNEL) The attached patch is an easy work-around for fixing the issue. However, it may not be the perfect appropriate change. Since I do not master MariaDB, I'd like someone to propose a fully correct fix. Thanks/Regards, Tony Reix tony.reix(a)atos.net ATOS / Bull SAS ATOS Expert IBM Coop Architect & Technical Leader Office : +33 (0) 4 76 29 72 67 1 rue de Provence - 38432 Échirolles - France www.atos.net

1 0

Re: [Maria-developers] 404 galera-arbitrator-3_25.3.26-xenial_amd64.deb
by Jongmin Kim 12 Apr '19

12 Apr '19

Ping to the Cc'ed folks, and add more Cc: our new mailing list (harukasan-ftp(a)lists.launchpad.net) On Thu, Apr 11, 2019 at 11:33:33PM +0900, Jongmin Kim wrote: > Hello, > > Thanks for reporting your failure! > > On Thu, Apr 11, 2019 at 03:51:17PM +0300, Sergei Shvarts wrote: > > Hi guys! > > > > looks like there is no galera-arbitraro-3 for xenial > > > > ERROR: unable to update: download errors: > > HTTP code 404 while fetching http://ftp.harukasan.org/mariadb/repo/10.3/ubuntu/pool/main/g/galera-3/gale… > > HTTP code 404 while fetching http://ftp.harukasan.org/mariadb/repo/10.3/ubuntu/pool/main/g/galera-3/gale… > > > > We digged some, and we found that: > - the package with 'Filename' is provided in Xenial 'Packages' [1], > - but the file is not exist at 'Filename' [2]. > We don't have idea about this issue. An official upstream mirror does have same issue [3,4]. > > [1] http://ftp.harukasan.org/mariadb/repo/10.3/ubuntu/dists/xenial/main/binary-… > [2] http://ftp.harukasan.org/mariadb/repo/10.3/ubuntu/pool/main/g/galera-3/ > [3] http://ftp.osuosl.org/pub/mariadb/repo/10.3/ubuntu/dists/xenial/main/binary… > [4] http://ftp.osuosl.org/pub/mariadb/repo/10.3/ubuntu/pool/main/g/galera-3/ > > So I'm forwarding this error to the package maintainer of 'galera-arbitrator-3': > - Codership Oy <info(a)codership.com> > > Also forwarding to the repository master of MariaDB repository: > - MariaDB Developers <maria-developers(a)lists.launchpad.net> > > Dear the package maintainer and MariaDB Developers, would you please dig this problem? > > Thank you! -- Jongmin Kim OpenPGP key located at https://jongmin.dev/pgp OpenPGP fingerprint: 012E 4A06 79E1 4EFC DAAE 9472 D39D 8D29 BAF3 6DF8

2 3

[Maria-developers] GSoC 2019 proposal
by Rajiv nayan 09 Apr '19

09 Apr '19

Hello. My name is Rajiv Nayan. I would like to contribute to mariaDB during GSoC 2019. More information can be found in my draft of proposal on the attached google doc. Thanks Proposal for "Implement UPDATE with result set, and Implement multiple-table UPDATE/DELETE returning a result set” for GSoC’19 https://docs.google.com/document/d/1vb9ixQlhOcHj4RpmaAuhMNNHw4jE_j1wjnzKuNW…

1 0

[Maria-developers] (no subject)
by Rajiv nayan 09 Apr '19

09 Apr '19

Hello. My name is Rajiv Nayan. I would like to contribute to mariaDB during GSoC 2019. More information can be found in my draft of proposal on the attached google doc. Thanks Proposal for "Implement UPDATE with result set, and Implement multiple-table UPDATE/DELETE returning a result set” for GSoC’19 https://docs.google.com/document/d/1vb9ixQlhOcHj4RpmaAuhMNNHw4jE_j1wjnzKuNW…

1 0

[Maria-developers] set_time() on slave and unversioned -> versioned replication
by Aleksey Midenkov 08 Apr '19

08 Apr '19

Hello, Sergei! In unversioned -> versioned scenario in the code below it first gets to Set time 4, creates some records (on slave) and then seconds on slave increased (X+1) while on master seconds are not yet increased (X). Then we get to Set time 3 and reset time on slave to X.0 therefore we get back in time and all stored records with timestamp X.n will be in future. 'n' came from ++system_time.sec_part in Set time 4. Why did you decided to use such logic of getting seconds from master and microseconds from slave? Since microseconds sooner or later reset to 0 it's not better than just assigning some random number. What sending microseconds from master conditionally only is good for? And for RBR since you don't send microseconds at all I see no good solution for this. inline void set_time(my_time_t t, ulong sec_part) { if (opt_secure_timestamp > (slave_thread ? SECTIME_REPL : SECTIME_SUPER)) { set_time(); // note that BINLOG itself requires SUPER DBUG_EXECUTE("time", print_start_time("Set time 1");); } else { if (sec_part <= TIME_MAX_SECOND_PART) { start_time= system_time.sec= t; start_time_sec_part= system_time.sec_part= sec_part; DBUG_EXECUTE("time", print_start_time("Set time 2");); } else if (t != system_time.sec) { DBUG_EXECUTE("time", print_system_time("System time"););; start_time= system_time.sec= t; start_time_sec_part= system_time.sec_part= 0; DBUG_EXECUTE("time", print_start_time("Set time 3");); } else { start_time= t; start_time_sec_part= ++system_time.sec_part; DBUG_EXECUTE("time", print_start_time("Set time 4");); } user_time.val= hrtime_from_time(start_time) + start_time_sec_part; PSI_CALL_set_thread_start_time(start_time); start_utime= utime_after_lock= microsecond_interval_timer(); } } -- All the best, Aleksey Midenkov @midenok

2 7

[Maria-developers] GSoC'19 proposal
by Rucha Deodhar 05 Apr '19

05 Apr '19

Hello. My name is Rucha Deodhar. I would like to contribute to mariaDB during GSoC 2019. More information can be found in my draft of proposal on the attached google doc. Any feedback would be appreciated. Thanks Proposal for “ADD RETURNING TO INSERT” for GSoC’19 <https://docs.google.com/document/d/1vE52kqZI-bhbRKbGBxeIrQKss8xXaGUU-0_PJHq…>

1 0

[Maria-developers] MDEV-18822: possible slowdown with optimizer trace
by Sergey Petrunia 04 Apr '19

04 Apr '19

Hi Varun, So I was thinking of the impact of the optimizer trace, and decided to look at how the most common check "if optimizer trace is enabled" is done. The code at SQL layer does this kind of check if (thd->trace_started()) (it is wrapped somewhere with "unlikely" but this is not consistent) THD::trace_started is defined as: bool THD::trace_started() { return opt_trace.is_started(); } which opt_trace here is Opt_trace_context. looking there: class Opt_trace_context { ... bool is_started() { return current_trace && is_enabled(); } following: bool Opt_trace_context::is_enabled() { if (current_trace) return current_trace->is_enabled(); return false; } current_trace is Opt_trace_stmt, looking there: bool Opt_trace_stmt::is_enabled() { return I_S_disabled == 0; } This is quite convoluted, and could probably made shorter. Disassembling sql_select.cc, I can see that the calls to thd->trace_started() are not inlined: if (unlikely(thd->trace_started())) 10ac9: 4c 89 ff mov rdi,r15 10acc: e8 00 00 00 00 call 10ad1 <_ZL32best_extension_by_limited_searchP4JOINyjddjjj+0x1a1> 10ad1: 84 c0 test al,al 10ad3: 0f 85 fc 02 00 00 jne 10dd5 <_ZL32best_extension_by_limited_searchP4JOINyjddjjj+0x4a5> if (unlikely(thd->trace_started())) 1197b: 4c 89 f7 mov rdi,r14 1197e: e8 00 00 00 00 call 11983 <_Z11choose_planP4JOINy+0x8a3> 11983: 84 c0 test al,al 11985: 0f 85 2f 01 00 00 jne 11aba <_Z11choose_planP4JOINy+0x9da> if (thd->trace_started()) 2d6f7: 48 8b bd b0 fc ff ff mov rdi,QWORD PTR [rbp-0x350] 2d6fe: e8 00 00 00 00 call 2d703 <_ZN4JOIN14optimize_innerEv+0x1963> 2d703: 84 c0 test al,al 2d705: 0f 85 6c 09 00 00 jne 2e077 <_ZN4JOIN14optimize_innerEv+0x22d7> if (thd->trace_started()) 2d7da: 48 8b bd b0 fc ff ff mov rdi,QWORD PTR [rbp-0x350] 2d7e1: e8 00 00 00 00 call 2d7e6 <_ZN4JOIN14optimize_innerEv+0x1a46> 2d7e6: 84 c0 test al,al 2d7e8: 0f 85 ff 1a 00 00 jne 2f2ed <_ZN4JOIN14optimize_innerEv+0x354d> But does it all matter? I think the action plan should be: 1. Reproduce the slowdown as it was reported in MDEV-18822. 2. If #1 succeeds in several re-runs, declare THD::trace_started as: class THD { public: bool trace_started; // initialize this to false in constructor and never touch it inline bool trace_started() const { return trace_started; } }; and then use search-replace to change "thd->trace_started()" to "unlikely(thd->trace_started)" and then re-run the benchmark. Any thoughts? BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

Re: [Maria-developers] [Commits] a1929d001c7: MDEV-18741: Optimizer trace: multi-part key ranges are printed incorrectly
by Sergey Petrunia 04 Apr '19

04 Apr '19

Hi Varun, In general, the patch moves in the right direction, but it is not yet there. Consider this example: create table ten(a int); insert into ten values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); create table t1 ( a int not null, b int not null, c int not null, d int not null, key(a,b,c) ); insert into t1 select a,a, a,a from ten; set optimizer_trace=1; explain select * from t1 where a between 1 and 4 and b < 50; mysql> select -> JSON_DETAILED(JSON_EXTRACT(trace, '$**.analyzing_range_alternatives')) -> from INFORMATION_SCHEMA.OPTIMIZER_TRACE; { "range_scan_alternatives": [ { "index": "a", "ranges": [ "(a) < (4,50)" ], "rowid_ordered": false, "using_mrr": false, "index_only": false, "rows": 1, "cost": 2.5002, "chosen": true } ], "analyzing_roworder_intersect": { "cause": "too few roworder scans" }, "analyzing_index_merge_union": [ ] } ] | while print_quick produces: T@18 : | | | | | | | | | | >print_quick quick range select, key a, length: 8 1 <= X < 4/50 other_keys: 0x0: T@18 : | | | | | | | | | | <print_quick Please investigate why this happens. On Wed, Apr 03, 2019 at 01:16:56PM +0530, Varun wrote: > revision-id: a1929d001c7b25e26d178834a196020bade819b8 (mariadb-10.3.6-318-ga1929d001c7) Any idea why does the commit trigger show "mariadb-10.3.." ? The patch is against 10.4, isnt it? > parent(s): 0dc442ac61ac7ff1a1d6bd7d6090c0f2251fb558 > author: Varun Gupta > committer: Varun Gupta > timestamp: 2019-04-03 13:10:07 +0530 > message: > > MDEV-18741: Optimizer trace: multi-part key ranges are printed incorrectly > > Changed the function append_range_all_keyparts to use sel_arg_range_seq_init / sel_arg_range_seq_next to produce ranges. > Also adjusted to print format for the ranges, now the ranges are printed as: > (keypart1_min, keypart2_min,..) OP (keypart1_name,keypart2_name, ..) OP (keypart1_max,keypart2_max, ..) > > Also added more tests for range and index merge access for optimizer trace ... > diff --git a/sql/opt_range.cc b/sql/opt_range.cc > index 5ab3d70214d..f53efc55158 100644 > --- a/sql/opt_range.cc > +++ b/sql/opt_range.cc > @@ -431,16 +431,18 @@ static int and_range_trees(RANGE_OPT_PARAM *param, > static bool remove_nonrange_trees(RANGE_OPT_PARAM *param, SEL_TREE *tree); > > static void print_key_value(String *out, const KEY_PART_INFO *key_part, > - const uchar *key); > + const uchar* key, uint length); > +static void print_keyparts_name(String *out, const KEY_PART_INFO *key_part, > + uint n_keypart, key_part_map keypart_map); Please fix indentation ^ > > static void append_range_all_keyparts(Json_writer_array *range_trace, > - String *range_string, > - String *range_so_far, const SEL_ARG *keypart, > + PARAM param, uint idx, > + SEL_ARG *keypart, > const KEY_PART_INFO *key_parts); > > static > -void append_range(String *out, const KEY_PART_INFO *key_parts, > - const uchar *min_key, const uchar *max_key, const uint flag); > +void append_range(String *out, const KEY_PART_INFO *key_part, > + KEY_MULTI_RANGE *range, uint n_key_parts); Please fix indentation ^ > > > /* > @@ -2273,10 +2275,7 @@ void TRP_RANGE::trace_basic_info(const PARAM *param, > // TRP_RANGE should not be created if there are no range intervals > DBUG_ASSERT(key); > > - String range_info; > - range_info.length(0); > - range_info.set_charset(system_charset_info); > - append_range_all_keyparts(&trace_range, NULL, &range_info, key, key_part); > + append_range_all_keyparts(&trace_range, *param, key_idx, key, key_part); > } > > > @@ -2489,10 +2488,8 @@ void TRP_GROUP_MIN_MAX::trace_basic_info(const PARAM *param, > // can have group quick without ranges > if (index_tree) > { > - String range_info; > - range_info.set_charset(system_charset_info); > - append_range_all_keyparts(&trace_range, NULL, &range_info, index_tree, > - key_part); > + append_range_all_keyparts(&trace_range, *param, param_idx, > + index_tree, key_part); > } > } > > @@ -6398,20 +6395,9 @@ void TRP_ROR_INTERSECT::trace_basic_info(const PARAM *param, > trace_isect_idx.add("rows", (*cur_scan)->records); > > Json_writer_array trace_range(thd, "ranges"); > - for (const SEL_ARG *current= (*cur_scan)->sel_arg->first(); current; > - current= current->next) > - { > - String range_info; > - range_info.set_charset(system_charset_info); > - for (const SEL_ARG *part= current; part; > - part= part->next_key_part ? part->next_key_part : nullptr) > - { > - const KEY_PART_INFO *cur_key_part= key_part + part->part; > - append_range(&range_info, cur_key_part, part->min_value, > - part->max_value, part->min_flag | part->max_flag); > - } > - trace_range.add(range_info.ptr(), range_info.length()); > - } > + > + append_range_all_keyparts(&trace_range, *param, (*cur_scan)->idx, > + (*cur_scan)->sel_arg->first(), key_part); Please fix indentation ^ > } > } > > @@ -7389,9 +7375,6 @@ static TRP_RANGE *get_key_scans_params(PARAM *param, SEL_TREE *tree, > const KEY &cur_key= param->table->key_info[keynr]; > const KEY_PART_INFO *key_part= cur_key.key_part; > > - String range_info; > - range_info.set_charset(system_charset_info); > - > index_scan->idx= idx; > index_scan->keynr= keynr; > index_scan->key_info= &param->table->key_info[keynr]; > @@ -7402,8 +7385,8 @@ static TRP_RANGE *get_key_scans_params(PARAM *param, SEL_TREE *tree, > *tree->index_scans_end++= index_scan; > > if (unlikely(thd->trace_started())) > - append_range_all_keyparts(&trace_range, NULL, &range_info, key, > - key_part); > + append_range_all_keyparts(&trace_range, (*param), idx, > + key, key_part); Please fix indentation ^ > trace_range.end(); > > trace_idx.add("rowid_ordered", param->is_ror_scan) > @@ -13554,11 +13537,8 @@ get_best_group_min_max(PARAM *param, SEL_TREE *tree, double read_time) > Json_writer_array trace_range(thd, "ranges"); > > const KEY_PART_INFO *key_part= cur_index_info->key_part; > - > - String range_info; > - range_info.set_charset(system_charset_info); > - append_range_all_keyparts(&trace_range, NULL, &range_info, > - cur_index_tree, key_part); > + append_range_all_keyparts(&trace_range, *param, cur_param_idx, > + cur_index_tree, key_part); Please fix indentation ^ > } > } > cost_group_min_max(table, cur_index_info, cur_used_key_parts, > @@ -15730,12 +15710,17 @@ void QUICK_GROUP_MIN_MAX_SELECT::dbug_dump(int indent, bool verbose) > } > > #endif /* !DBUG_OFF */ > + > static > void append_range(String *out, const KEY_PART_INFO *key_part, > - const uchar *min_key, const uchar *max_key, const uint flag) > + KEY_MULTI_RANGE *range, uint n_key_parts) > { > - if (out->length() > 0) > - out->append(STRING_WITH_LEN(" AND ")); > + uint flag= range->range_flag; > + String key_name; > + key_name.set_charset(system_charset_info); > + key_part_map keypart_map= range->start_key.length ? > + range->start_key.keypart_map : > + range->end_key.keypart_map; > > if (flag & GEOM_FLAG) > { > @@ -15744,22 +15729,24 @@ void append_range(String *out, const KEY_PART_INFO *key_part, > range types, so printing "col < some_geom" doesn't make sense. > Just print the column name, not operator. > */ > - out->append(key_part->field->field_name); > + print_keyparts_name(out, key_part, n_key_parts, keypart_map); > out->append(STRING_WITH_LEN(" ")); > - print_key_value(out, key_part, min_key); > + print_key_value(out, key_part, range->start_key.key, > + range->start_key.length); > return; > } > > if (!(flag & NO_MIN_RANGE)) > { > - print_key_value(out, key_part, min_key); > + print_key_value(out, key_part, range->start_key.key, > + range->start_key.length); > if (flag & NEAR_MIN) > out->append(STRING_WITH_LEN(" < ")); > else > out->append(STRING_WITH_LEN(" <= ")); > } > > - out->append(key_part->field->field_name); > + print_keyparts_name(out, key_part, n_key_parts, keypart_map); > > if (!(flag & NO_MAX_RANGE)) > { > @@ -15767,7 +15754,8 @@ void append_range(String *out, const KEY_PART_INFO *key_part, > out->append(STRING_WITH_LEN(" < ")); > else > out->append(STRING_WITH_LEN(" <= ")); > - print_key_value(out, key_part, max_key); > + print_key_value(out, key_part, range->end_key.key, > + range->end_key.length); > } > } > > @@ -15775,60 +15763,39 @@ void append_range(String *out, const KEY_PART_INFO *key_part, > > Add ranges to the trace > For ex: > - query: select * from t1 where a=2 ; > - and we have an index on a , so we create a range > - 2 <= a <= 2 > + lets say we have an index a_b(a,b) > + query: select * from t1 where a=2 and b=4 ; > + so we create a range: > + (2,4) <= (a,b) <= (2,4) > this is added to the trace > */ > > static void append_range_all_keyparts(Json_writer_array *range_trace, > - String *range_string, > - String *range_so_far, const SEL_ARG *keypart, > + PARAM param, uint idx, > + SEL_ARG *keypart, > const KEY_PART_INFO *key_parts) > { > - > - DBUG_ASSERT(keypart); > - DBUG_ASSERT(keypart && keypart != &null_element); > - > - // Navigate to first interval in red-black tree > + SEL_ARG_RANGE_SEQ seq; > + KEY_MULTI_RANGE range; > + range_seq_t seq_it; > + uint flags= 0; > + RANGE_SEQ_IF seq_if = {NULL, sel_arg_range_seq_init, > + sel_arg_range_seq_next, 0, 0}; > + KEY *keyinfo= param.table->key_info + param.real_keynr[idx]; > + uint n_key_parts= param.table->actual_n_key_parts(keyinfo); > + seq.keyno= idx; > + seq.real_keyno= param.real_keynr[idx]; > + seq.param= &param; > + seq.start= keypart; > const KEY_PART_INFO *cur_key_part= key_parts + keypart->part; > - const SEL_ARG *keypart_range= keypart->first(); > - const size_t save_range_so_far_length= range_so_far->length(); > - > + seq_it= seq_if.init((void *) &seq, 0, flags); > > - while (keypart_range) > + while (!seq_if.next(seq_it, &range)) > { > - // Append the current range predicate to the range String > - switch (keypart->type) > - { > - case SEL_ARG::Type::KEY_RANGE: > - append_range(range_so_far, cur_key_part, keypart_range->min_value, > - keypart_range->max_value, > - keypart_range->min_flag | keypart_range->max_flag); > - break; > - case SEL_ARG::Type::MAYBE_KEY: > - range_so_far->append("MAYBE_KEY"); > - break; > - case SEL_ARG::Type::IMPOSSIBLE: > - range_so_far->append("IMPOSSIBLE"); > - break; > - default: > - DBUG_ASSERT(false); > - break; > - } > - > - if (keypart_range->next_key_part && > - keypart_range->next_key_part->part == > - keypart_range->part + 1 && > - keypart_range->is_singlepoint()) > - { > - append_range_all_keyparts(range_trace, range_string, range_so_far, > - keypart_range->next_key_part, key_parts); > - } > - else > - range_trace->add(range_so_far->c_ptr_safe(), range_so_far->length()); > - keypart_range= keypart_range->next; > - range_so_far->length(save_range_so_far_length); > + String range_info; > + range_info.set_charset(system_charset_info); > + append_range(&range_info, cur_key_part, &range, n_key_parts); > + range_trace->add(range_info.c_ptr_safe(), range_info.length()); > } > } > > @@ -15838,70 +15805,110 @@ static void append_range_all_keyparts(Json_writer_array *range_trace, > @param[out] out String the key is appended to > @param[in] key_part Index components description > @param[in] key Key tuple > + @param[in] used_length length of the key tuple > */ > + > static void print_key_value(String *out, const KEY_PART_INFO *key_part, > - const uchar *key) > + const uchar* key, uint used_length) > { > + out->append(STRING_WITH_LEN("(")); > Field *field= key_part->field; > + StringBuffer<128> tmp(system_charset_info); > + TABLE *table= field->table; > + uint store_length; > + my_bitmap_map *old_sets[2]; > + dbug_tmp_use_all_columns(table, old_sets, table->read_set, table->write_set); > + const uchar *key_end= key+used_length; > > - if (field->flags & BLOB_FLAG) > + for (; key < key_end; key+=store_length, key_part++) > { > - // Byte 0 of a nullable key is the null-byte. If set, key is NULL. > - if (field->real_maybe_null() && *key) > - out->append(STRING_WITH_LEN("NULL")); > - else > - (field->type() == MYSQL_TYPE_GEOMETRY) > - ? out->append(STRING_WITH_LEN("unprintable_geometry_value")) > - : out->append(STRING_WITH_LEN("unprintable_blob_value")); > - return; > - } > + field= key_part->field; > + store_length= key_part->store_length; > + if (field->flags & BLOB_FLAG) > + { > + // Byte 0 of a nullable key is the null-byte. If set, key is NULL. > + if (field->real_maybe_null() && *key) > + out->append(STRING_WITH_LEN("NULL")); > + else > + (field->type() == MYSQL_TYPE_GEOMETRY) > + ? out->append(STRING_WITH_LEN("unprintable_geometry_value")) > + : out->append(STRING_WITH_LEN("unprintable_blob_value")); > + goto next; > + } > > - uint store_length= key_part->store_length; > + if (field->real_maybe_null()) > + { > + /* > + Byte 0 of key is the null-byte. If set, key is NULL. > + Otherwise, print the key value starting immediately after the > + null-byte > + */ > + if (*key) > + { > + out->append(STRING_WITH_LEN("NULL")); > + goto next; > + } > + key++; // Skip null byte > + store_length--; > + } > > - if (field->real_maybe_null()) > - { > /* > - Byte 0 of key is the null-byte. If set, key is NULL. > - Otherwise, print the key value starting immediately after the > - null-byte > + Binary data cannot be converted to UTF8 which is what the > + optimizer trace expects. If the column is binary, the hex > + representation is printed to the trace instead. > */ > - if (*key) > + if (field->flags & BINARY_FLAG) > { > - out->append(STRING_WITH_LEN("NULL")); > - return; > + out->append("0x"); > + for (uint i = 0; i < store_length; i++) > + { > + out->append(_dig_vec_lower[*(key + i) >> 4]); > + out->append(_dig_vec_lower[*(key + i) & 0x0F]); > + } > + goto next; > } > - key++; // Skip null byte > - store_length--; > + > + field->set_key_image(key, key_part->length); > + if (field->type() == MYSQL_TYPE_BIT) > + (void)field->val_int_as_str(&tmp, 1); // may change tmp's charset > + else > + field->val_str(&tmp); // may change tmp's charset > + out->append(tmp.ptr(), tmp.length(), tmp.charset()); > + > + next: > + if (key + store_length < key_end) > + out->append(STRING_WITH_LEN(",")); > } > + dbug_tmp_restore_column_maps(table->read_set, table->write_set, old_sets); > + out->append(STRING_WITH_LEN(")")); > +} > > - /* > - Binary data cannot be converted to UTF8 which is what the > - optimizer trace expects. If the column is binary, the hex > - representation is printed to the trace instead. > - */ > - if (field->flags & BINARY_FLAG) > +/** > + Print key parts involed in a range > + @param[out] out String the key is appended to > + @param[in] key_part Index components description > + @param[in] n_keypart Number of keyparts in index > + @param[in] keypart_map map for keyparts involved in the range > +*/ > + > +void print_keyparts_name(String *out, const KEY_PART_INFO *key_part, > + uint n_keypart, key_part_map keypart_map) > +{ > + uint i; > + out->append(STRING_WITH_LEN("(")); > + bool first_keypart= TRUE; > + for (i=0; i < n_keypart; key_part++, i++) > { > - out->append("0x"); > - for (uint i = 0; i < store_length; i++) > + if (keypart_map & (1 << i)) > { > - out->append(_dig_vec_lower[*(key + i) >> 4]); > - out->append(_dig_vec_lower[*(key + i) & 0x0F]); > + if (first_keypart) > + first_keypart= FALSE; > + else > + out->append(STRING_WITH_LEN(",")); > + out->append(key_part->field->field_name); > } > - return; > + else > + break; > } > - > - StringBuffer<128> tmp(system_charset_info); > - TABLE *table= field->table; > - my_bitmap_map *old_sets[2]; > - > - dbug_tmp_use_all_columns(table, old_sets, table->read_set, table->write_set); > - > - field->set_key_image(key, key_part->length); > - if (field->type() == MYSQL_TYPE_BIT) > - (void)field->val_int_as_str(&tmp, 1); // may change tmp's charset > - else > - field->val_str(&tmp); // may change tmp's charset > - out->append(tmp.ptr(), tmp.length(), tmp.charset()); > - > - dbug_tmp_restore_column_maps(table->read_set, table->write_set, old_sets); > -} > + out->append(STRING_WITH_LEN(")")); > +} BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

Re: [Maria-developers] 1581f65be57: MDEV-13301 Optimize DROP INDEX, ADD INDEX into RENAME INDEX
by Sergei Golubchik 02 Apr '19

02 Apr '19

Hi, Eugene! See the review below. This looked pretty much ok, a couple of style comments. And why did you not implement the new ALTER TABLE .. .RENAME INDEX syntax? I agree that a pair of DROP/ADD should act as RENAME. But a dedicated RENAME would be also good to have, at least for compatibility reasons. On Apr 01, Eugene Kosov wrote: > revision-id: 1581f65be57 (mariadb-10.4.3-69-g1581f65be57) > parent(s): e012d266807 > author: Eugene Kosov <claprix(a)yandex.ru> > committer: Eugene Kosov <claprix(a)yandex.ru> > timestamp: 2019-03-15 18:44:05 +0300 > message: > > MDEV-13301 Optimize DROP INDEX, ADD INDEX into RENAME INDEX > > Just rename index in data dictionary and in InnoDB cache when it's possible. > Introduce ALTER_INDEX_RENAME for that purpose so that engines can optimize > such operation. > > Unused code between macro MYSQL_RENAME_INDEX was removed. > > compare_keys_but_name(): compare index definitions except for index names > > Alter_inplace_info::rename_keys: > ha_innobase_inplace_ctx::rename_keys: vector of rename indexes > > fill_alter_inplace_info():: fills Alter_inplace_info::rename_keys > > diff --git a/sql/sql_table.cc b/sql/sql_table.cc > index e1337a0f710..e1131422117 100644 > --- a/sql/sql_table.cc > +++ b/sql/sql_table.cc > @@ -7062,6 +7061,43 @@ static bool fill_alter_inplace_info(THD *thd, TABLE *table, bool varchar, > new_key->option_struct; > } > > + uint &add_count= ha_alter_info->index_add_count; Coding style: no modifiable references please. write ha_alter_info->index_add_count++ same below > + for (uint i= 0; i < add_count; i++) > + { > + uint *&add_buffer= ha_alter_info->index_add_buffer; > + const KEY *new_key= ha_alter_info->key_info_buffer + add_buffer[i]; > + > + uint &drop_count= ha_alter_info->index_drop_count; > + for (uint j= 0; j < drop_count; j++) > + { > + KEY **&drop_buffer= ha_alter_info->index_drop_buffer; > + const KEY *old_key= drop_buffer[j]; > + > + if (!lex_string_cmp(system_charset_info, &old_key->name, &new_key->name)) > + continue; Why? I'd think and if it's Compare_keys::Equal and the name is the same too, you can simply remove add/drop elements without adding a rename element. > + > + if (Compare_keys::Equal != compare_keys_but_name(old_key, new_key, > + alter_info, table, > + new_pk, old_pk)) Coding style: don't invert comparisons > + { > + continue; > + } > + > + ha_alter_info->handler_flags|= ALTER_RENAME_INDEX; > + ha_alter_info->rename_keys.push_back( > + Alter_inplace_info::Rename_key_pair(old_key, new_key)); > + > + memcpy(add_buffer + i, add_buffer + i + 1, > + sizeof(add_buffer[0]) * (add_count - i - 1)); > + memcpy(drop_buffer + j, drop_buffer + j + 1, > + sizeof(drop_buffer[0]) * (drop_count - j - 1)); > + --add_count; > + --drop_count; > + --i; // this index once again > + break; > + } > + } > + Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

2 3

Re: [Maria-developers] 126a9b6db25: MDEV-18875 Assertion `thd->transaction.stmt.ha_list == __null || trans == &thd->transaction.stmt' failed or bogus ER_DUP_ENTRY upon ALTER TABLE with versioning
by Sergei Golubchik 02 Apr '19

02 Apr '19

Hi, Nikita! On Apr 02, Nikita Malyavin wrote: > revision-id: 126a9b6db25 (mariadb-10.3.12-101-g126a9b6db25) > parent(s): 1c60f40868f > author: Nikita Malyavin <nikitamalyavin(a)gmail.com> > committer: Nikita Malyavin <nikitamalyavin(a)gmail.com> > timestamp: 2019-03-23 03:37:10 +1000 > message: > > MDEV-18875 Assertion `thd->transaction.stmt.ha_list == __null || trans == &thd->transaction.stmt' failed or bogus ER_DUP_ENTRY upon ALTER TABLE with versioning > > diff --git a/sql/handler.cc b/sql/handler.cc > index e01392d236c..9d295afb631 100644 > --- a/sql/handler.cc > +++ b/sql/handler.cc > @@ -1420,7 +1420,8 @@ int ha_commit_trans(THD *thd, bool all) > > #if 1 // FIXME: This should be done in ha_prepare(). > if (rw_trans || (thd->lex->sql_command == SQLCOM_ALTER_TABLE && > - thd->lex->alter_info.flags & ALTER_ADD_SYSTEM_VERSIONING)) > + thd->lex->alter_info.flags & ALTER_ADD_SYSTEM_VERSIONING && > + is_real_trans)) Why there's a special check for SQLCOM_ALTER_TABLE + ALTER_ADD_SYSTEM_VERSIONING in the first place? Does it mean that ALTER TABLE ... ADD SYSTEM VERSIONING does not marks a transaction as read/write? That's be weird. Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

2 2

Re: [Maria-developers] 5469d88e7b4: MDEV-19049 Server crashes in check_duplicate_long_entry_key, ASAN stack-buffer-overflow in Field_blob::get_key_image
by Sergei Golubchik 02 Apr '19

02 Apr '19

Hi, Sachin! On Apr 02, Sachin Setiya wrote: > revision-id: 5469d88e7b4 (mariadb-10.4.3-104-g5469d88e7b4) > parent(s): 0bc42602266 > author: Sachin <sachin.setiya(a)mariadb.com> > committer: Sachin <sachin.setiya(a)mariadb.com> > timestamp: 2019-03-28 11:29:25 +0530 > message: > > MDEV-19049 Server crashes in check_duplicate_long_entry_key, ASAN stack-buffer-overflow in Field_blob::get_key_image > > Long Unique keys should always be last unique key. Why? > diff --git a/sql/sql_table.cc b/sql/sql_table.cc > index ad62ecc1103..c755a74e174 100644 > --- a/sql/sql_table.cc > +++ b/sql/sql_table.cc > @@ -2796,6 +2797,14 @@ static int sort_keys(KEY *a, KEY *b) > { > if (!(b_flags & HA_NOSAME)) > return -1; > + /* > + Long Unique keys should always be last unique key. > + Before this patch they used to change order wrt to partial keys (MDEV-19049) don't write the code history "before this patch it worked like that" in the code itself. When you look at the code there is no "patch", just a comment. The concepts of "patch" and "before it used to be" belongs to the history, and logically should be in the commit comment, not in a code comment. > + */ > + if (a->algorithm == HA_KEY_ALG_LONG_HASH) > + return 1; > + if (b->algorithm == HA_KEY_ALG_LONG_HASH) > + return -1; > if ((a_flags ^ b_flags) & HA_NULL_PART_KEY) > { > /* Sort NOT NULL keys before other keys */ > Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

1 0

[Maria-developers] GSOC 2019 proposal draft
by Алексей Могилёвкин 02 Apr '19

02 Apr '19

Hello. My name is Alexey Mogilyovkin. I would like to contribute to mariadb during GSOC 2019. More information can be found in my draft of proposal on the attached google doc. Any feedback would be appreciated. MariaDB proposal <https://docs.google.com/document/d/12utv0vUoV2FfPMSH2fOR3lrYXT_mS5er_S1XVuM…>

3 2

[Maria-developers] Buildbot image update request
by Otto Kekäläinen 01 Apr '19

01 Apr '19

Hello! Can you please update the image used by test kvm-deb-jessie-ppc64le so that the 'apt-get update' is fresh and package lists are recent? It would fix the error in http://buildbot.askmonty.org/buildbot/builders/kvm-deb-jessie-ppc64le/build… E: Failed to fetch http://ftp.us.debian.org/debian/pool/main/m/mpfr4/libmpfr4_3.1.2-2_ppc64el.… 404 Not Found [IP: 64.50.233.100 80] E: Failed to fetch http://ftp.us.debian.org/debian/pool/main/g/gcc-4.9/libatomic1_4.9.2-10_ppc… 404 Not Found [IP: 64.50.233.100 80] E: Failed to fetch http://ftp.us.debian.org/debian/pool/main/r/rsync/rsync_3.1.1-3_ppc64el.deb 404 Not Found [IP: 64.50.233.100 80] E: Failed to fetch http://ftp.us.debian.org/debian/pool/main/s/socat/socat_1.7.2.4-2_ppc64el.d… 404 Not Found [IP: 64.50.233.100 80] E: Failed to fetch http://ftp.us.debian.org/debian/pool/main/libd/libdbd-mysql-perl/libdbd-mys… 404 Not Found [IP: 64.50.233.100 80] Also, can you please run 'apt-get install libzstd-dev' in the image used in arm64-xenial-deb? It would fix http://buildbot.askmonty.org/buildbot/builders/arm64-xenial-deb/builds/2973… dpkg-checkbuilddeps: error: Unmet build dependencies: libzstd-dev dpkg-buildpackage: warning: build dependencies/conflicts unsatisfied; aborting Thanks!

2 1

Re: [Maria-developers] required mysql_upgrade; MDEV-14637
by Michal Schorm 01 Apr '19

01 Apr '19

On Thu, Mar 28, 2019 at 11:16 AM Michal Schorm <mschorm(a)redhat.com> wrote: > > Hello, > I have a questions regarding the warnings showing up at 10.2 and 10.3 releases: > > When upgrading from MariaDB 10.2.16 or earlier to MariaDB 10.2.17 or higher, > running mysql_upgrade is required due to changes introduced in MDEV-14637. > > When upgrading from MariaDB 10.3.8 or earlier to MariaDB 10.3.9 or higher, > running mysql_upgrade is required due to changes introduced in MDEV-14637. > > -- > > 1) The same applies for upgrading <=10.2.16 to >= 10.3.9, right? > > 2) Does it mean, it is impossible to use "dump and restore" technique, > or it only means I need to run the mysql_upgrade afterwards? > > 3) The problem is not triggered when upgrading from 5.5, 10.0 and 10.1? > 4) The problem is not triggered when upgrading to 10.4 ? I found a reply in the mailing list archive which haven't reached my inbox: From: Reindl Harald Date: Thu, 28 Mar 2019 11:59:50 +0100 > just run "mysql_upgrade" after each update and you are done > our production datadirs date back to 2002, originally on windows then > moved to MacOS and in 2008 to Fedora and i did not dump/restore a single > time in my whole life, this is not postgresql My question is focused on the clarification in our downstream documentation. So while usually I'd be fine with personal experience, in this case, I'd prefer more in-depth answer covering. I'm forwarding this mail to devel mailing list. -- Michal Schorm Software Engineer Core Services - Databases Team Red Hat --

1 0

Re: [Maria-developers] 9976699033c: Test fixes (versioning.replace, versioning.foreign)
by Sergei Golubchik 31 Mar '19

31 Mar '19

Hi, Aleksey! On Mar 27, Aleksey Midenkov wrote: > revision-id: 9976699033c (mariadb-10.3.7-30-g9976699033c) > parent(s): c0f97710582 > author: Aleksey Midenkov <midenok(a)gmail.com> > committer: Aleksey Midenkov <midenok(a)gmail.com> > timestamp: 2018-05-30 13:12:47 +0300 > message: > > Test fixes (versioning.replace, versioning.foreign) > > * replace.test: unwrapped create_table procedure; > * foreign.test: use key_type combinations. > > diff --git a/mysql-test/suite/versioning/key_type.combinations b/mysql-test/suite/versioning/key_type.combinations > index 1929aee9a84..fb3888d3d0f 100644 > --- a/mysql-test/suite/versioning/key_type.combinations > +++ b/mysql-test/suite/versioning/key_type.combinations > @@ -1,2 +1,3 @@ > [unique] > -[pk] > +[primary] > +[secondary] generally prefer shorter combination names, otherwise mtr output becomes garbled and difficult to read. > diff --git a/mysql-test/suite/versioning/primary_or_unique.inc b/mysql-test/suite/versioning/primary_or_unique.inc > new file mode 100644 > index 00000000000..199d8bab942 > --- /dev/null > +++ b/mysql-test/suite/versioning/primary_or_unique.inc > @@ -0,0 +1,5 @@ > +--source suite/versioning/key_type.inc > +--require suite/versioning/primary_or_unique.require don't do .require, please, it's a technique from 1998, before mysqltest had the "skip" command. Use if (...) { skip; } > +disable_query_log; > +eval select "$KEY_TYPE" != "key" as pk_or_unique; > +enable_query_log; > diff --git a/mysql-test/suite/versioning/t/engines.combinations b/mysql-test/suite/versioning/t/engines.combinations > deleted file mode 100644 > index 561c5656929..00000000000 > --- a/mysql-test/suite/versioning/t/engines.combinations > +++ /dev/null > @@ -1,8 +0,0 @@ > -[timestamp] > -default-storage-engine=innodb > - > -[trx_id] > -default-storage-engine=innodb > - > -[myisam] > -default-storage-engine=myisam Is that intentional? I thought many tests use it to run in different versioning modes. > diff --git a/mysql-test/suite/versioning/t/foreign.test b/mysql-test/suite/versioning/t/foreign.test > index 566d481c2a8..72e5b047cbb 100644 > --- a/mysql-test/suite/versioning/t/foreign.test > +++ b/mysql-test/suite/versioning/t/foreign.test > @@ -1,11 +1,14 @@ > +--source suite/versioning/primary_only.inc > --source suite/versioning/common.inc what's the point of this? if you're only allowing primary key here, you can just as well write manually "primary key" in all tests, without $KEY_TYPE and replace_result. Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

2 1

[Maria-developers] EITS member variables in both TABLE and TABLE_SHARE?
by Sergey Petrunia 29 Mar '19

29 Mar '19

Hello Igor, TABLE_SHARE has TABLE_STATISTICS_CB, which has these members: struct TABLE_STATISTICS_CB { ... bool stats_is_read; /* Statistical data for table has been read from statistical tables */ bool histograms_are_read; } At the same time, TABLE has: struct TABLE { ... bool stats_is_read; /* Persistent statistics is read for the table */ bool histograms_are_read; I would like to understand why both structures need these variables? Is there any difference in meaning between them? BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

Re: [Maria-developers] 74b2eba1ca6: MDEV-15458 Segfault in heap_scan() upon UPDATE after ADD SYSTEM VERSIONING
by Sergei Golubchik 29 Mar '19

29 Mar '19

Hi, Aleksey! On Mar 26, Aleksey Midenkov wrote: > revision-id: 74b2eba1ca6 (mariadb-10.3.12-115-g74b2eba1ca6) > parent(s): 2c0901d808b > author: Aleksey Midenkov <midenok(a)gmail.com> > committer: Sergei Golubchik <serg(a)mariadb.com> > timestamp: 2019-03-26 17:43:38 +0100 > message: > > MDEV-15458 Segfault in heap_scan() upon UPDATE after ADD SYSTEM VERSIONING > > * HA_EXTRA_REMEMBER_POS, HA_EXTRA_RESTORE_POS for HEAP storage engine > * Versioning tests support > > diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc > index abe4254db3e..1d51947cc86 100644 > --- a/sql/sql_insert.cc > +++ b/sql/sql_insert.cc > @@ -1643,7 +1643,12 @@ int vers_insert_history_row(TABLE *table) > if (row_start->cmp(row_start->ptr, row_end->ptr) >= 0) > return 0; > > - return table->file->ha_write_row(table->record[0]); > + int res; > + if ((res= table->file->extra(HA_EXTRA_REMEMBER_POS))) > + return res; > + if ((res= table->file->ha_write_row(table->record[0]))) > + return res; > + return table->file->extra(HA_EXTRA_RESTORE_POS); > } Frankly, I don't like it. I mean, this particular fix is fine, but here's the problem: We now have (at least) three features where the server might need to do something (insert/update/search) during the index/rnd scan. And three different solutions: you use HA_EXTRA_REMEMBER_POS in system versioning, Nikita simply sets delete_while_scanning=false in application time period tables, and Sachin uses handler::clone() to create a second handler that can be used to check for hash collisions without disrupting the scan on the primary handler. I think it's getting somewhat out of hands. Can we please reduce the number of approaches to fix the same issue? Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

4 4

Re: [Maria-developers] ab6ef429a84: MDEV-16370 row-based binlog events (updates by primary key) can not be applied multiple times to system versioned tables
by Sergei Golubchik 29 Mar '19

29 Mar '19

Hi, Aleksey! On Jan 31, Aleksey Midenkov wrote: > revision-id: ab6ef429a84 (versioning-1.0.7-2-gab6ef429a84) > parent(s): 14c2f90ad08 > author: Aleksey Midenkov <midenok(a)gmail.com> > committer: Aleksey Midenkov <midenok(a)gmail.com> > timestamp: 2018-11-26 12:46:37 +0300 > message: > > MDEV-16370 row-based binlog events (updates by primary key) can not be applied multiple times to system versioned tables > > Case 2: > > * Idempotent BINLOG updates: duplicate key on history row is skipped; > * Fix row start and warn when applying BINLOG from the past. As far as I can see now you cannot just arbitrarily adjust row_start. Think of it in term or application time period tables, a primary key is (kind of) defined WITHOUT OVERLAPS. There is no explicit constraint, it happens automatically when the history is valid. Now, here's your test case: timestamp=100; insert (1,1) -> the table has (1,1,100,MAX); timestamp=300; update to (1,3) -> (1,1,100,300),(1,3,300,MAX); timestamp=200; update to (1,2) -> here the hisory row should've been (1,3,300,200) but you adjust it to get 1,1,100,300 1,3,199,200 1,2,200,MAX and if you select AS OF 199, you'll get two rows for the pk=1 So, let's * ignore HA_ERR_FOUND_DUPP_KEY (please check if it's needed for insert or delete too) * add TIMESTAMP=xxx to mysqldumps. can vers_insert_history_row see row_start>row_end in a normal use case? no replication or any other way of adjusting the timestamp. equality can surely happen, but greater than ? Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

2 1

[Maria-developers] THD::set_time()
by Aleksey Midenkov 28 Mar '19

28 Mar '19

What if system_time.sec_part == TIME_MAX_SECOND_PART here: else { start_time= t; start_time_sec_part= ++system_time.sec_part; } ? -- All the best, Aleksey Midenkov @midenok

1 0

[Maria-developers] (no subject)
by Samyak Jain 28 Mar '19

28 Mar '19

I want to contribute for full outer join gsoc project of mariaDB. Please provide the technical requirements for this and how I start contributing on this

1 0

Re: [Maria-developers] c1bf5766aed: MDEV-16490: It's possible to make a system versioned table without any versioning field
by Sergei Golubchik 27 Mar '19

27 Mar '19

Hi, Nikita! On Mar 27, Nikita Malyavin wrote: > revision-id: c1bf5766aed (versioning-1.0.5-41-gc1bf5766aed) > parent(s): 141a5b24843 > author: Nikita Malyavin <nikitamalyavin(a)gmail.com> > committer: Nikita Malyavin <nikitamalyavin(a)gmail.com> > timestamp: 2018-07-20 23:37:26 +1000 > message: > > MDEV-16490: It's possible to make a system versioned table without any versioning field > > * do not allow versioned table to be without versioned (non-system) fields > * prohibit changing field versioning, when removing table versioning > > diff --git a/sql/sql_table.cc b/sql/sql_table.cc > index dbf400b53d7..ee4cf457f45 100644 > --- a/sql/sql_table.cc > +++ b/sql/sql_table.cc > @@ -8473,11 +8473,21 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, > } > } > > - if (table->versioned() && !(alter_info->flags & ALTER_DROP_SYSTEM_VERSIONING) && > - new_create_list.elements == VERSIONING_FIELDS) > + if (table->versioned()) > { > - my_error(ER_VERS_TABLE_MUST_HAVE_COLUMNS, MYF(0), table->s->table_name.str); > - goto err; > + uint versioned_fields= 0; > + field_it.rewind(); > + while (Create_field *f= field_it++) > + { > + if (!(f->flags & VERS_UPDATE_UNVERSIONED_FLAG)) > + versioned_fields++; > + } > + if (!(alter_info->flags & ALTER_DROP_SYSTEM_VERSIONING) && > + versioned_fields == VERSIONING_FIELDS) > + { > + my_error(ER_VERS_TABLE_MUST_HAVE_COLUMNS, MYF(0), table->s->table_name.str); > + goto err; > + } wouldn't that fit naturally into Vers_parse_info::fix_alter_info() ? Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

1 0

Re: [Maria-developers] [Commits] e0fa1993692: MDEV-18899: Server crashes in Field::set_warning_truncated_wrong_value
by Sergey Petrunia 27 Mar '19

27 Mar '19

Hi Varun, First, Some input on the test: On Wed, Mar 27, 2019 at 01:55:16PM +0530, Varun wrote: > revision-id: e0fa1993692a3552feddae56cbee3078d4da2121 (mariadb-10.2.22-91-ge0fa1993692) > parent(s): 50a8fc52988d13a5164a1a542b9d7a85e3ecc1c1 > author: Varun Gupta > committer: Varun Gupta > timestamp: 2019-03-27 13:48:44 +0530 > message: > > MDEV-18899: Server crashes in Field::set_warning_truncated_wrong_value > > To fix the crash there we need to make sure that the > server while storing the statistical values in statistical tables should do it > in a multi-byte safe way. > Also there is no need to throw warnings if there is truncation while storing > values from statistical fields. > ... > --- a/mysql-test/r/stat_tables_innodb.result > +++ b/mysql-test/r/stat_tables_innodb.result > @@ -651,6 +651,63 @@ SELECT MAX(pk) FROM t1; > MAX(pk) > NULL > DROP TABLE t1; > +# > +# MDEV-18899: Server crashes in Field::set_warning_truncated_wrong_value > +# > +set names utf8; > +set @save_optimizer_use_condition_selectivity=@@optimizer_use_condition_selectivity; > +set optimizer_use_condition_selectivity=4; > +set use_stat_tables=preferably; > +set @save_histogram_size= @@histogram_size; > +set histogram_size=255; > +create table t1 ( a varchar(255) character set utf8); > +insert into t1 values (REPEAT('ӥ',255)), (REPEAT('ç',255)); > +analyze table t1; > +Table Op Msg_type Msg_text > +test.t1 analyze status Engine-independent statistics collected > +test.t1 analyze status OK > +select HEX(RIGHT(min_value, 1)), length(min_value) from mysql.column_stats; > +HEX(RIGHT(min_value, 1)) length(min_value) > +A7 254 > +select HEX(RIGHT(max_value, 1)), length(max_value) from mysql.column_stats; > +HEX(RIGHT(max_value, 1)) length(max_value) > +A5 254 > +analyze select * from t1 where a >= 'ӥ'; > +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra > +1 SIMPLE t1 ALL NULL NULL NULL NULL 2 2.00 50.00 50.00 Using where > +set @save_sql_mode= @@sql_mode; > +set sql_mode='ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION'; > +update mysql.column_stats set min_value= REPEAT('ӥ',255); This looks scary, what if there were some other data in these tables? Can you change add a WHERE db_name=database() and table_name='t1' ? > +Warnings: > +Warning 1265 Data truncated for column 'min_value' at row 1 > +select HEX(RIGHT(min_value, 1)), length(min_value) from mysql.column_stats; > +HEX(RIGHT(min_value, 1)) length(min_value) > +D3 255 > +analyze select * from t1 where a >= 'ӥ'; > +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra > +1 SIMPLE t1 ALL NULL NULL NULL NULL 2 2.00 50.00 50.00 Using where > +set names latin1; > +drop table t1; > +CREATE TABLE t1 (col1 date); > +INSERT INTO t1 VALUES('2004-01-01'),('2004-02-29'); > +INSERT INTO t1 VALUES('0000-10-31'); > +analyze table t1; > +Table Op Msg_type Msg_text > +test.t1 analyze status Engine-independent statistics collected > +test.t1 analyze status OK > +update mysql.column_stats set min_value='2004-0-31123'; The same as above. > +select min_value from mysql.column_stats; > +min_value > +2004-0-31123 This case also makes one wonder whether not producing a warning was a good idea after all. A varchar string that is 255-byte long, and after 255 bytes it gets truncated in the middle of a character is a rare case. An incorrect datetime value... not as rare? > +select * from t1; Is the above supposed to read the EITS data? It doesn't as the query doesn't use the WHERE clause. > +col1 > +2004-01-01 > +2004-02-29 > +0000-10-31 > +drop table t1; > +set @@sql_mode= @save_sql_mode; > set use_stat_tables=@save_use_stat_tables; > +set @@histogram_size= @save_histogram_size; > +set @@optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; > set optimizer_switch=@save_optimizer_switch_for_stat_tables_test; > SET SESSION STORAGE_ENGINE=DEFAULT; > diff --git a/mysql-test/t/stat_tables.test b/mysql-test/t/stat_tables.test > index b89ab2bbd2d..114c5c97a7e 100644 > --- a/mysql-test/t/stat_tables.test > +++ b/mysql-test/t/stat_tables.test > @@ -401,4 +401,44 @@ SELECT MAX(pk) FROM t1; > > DROP TABLE t1; > > +--echo # > +--echo # MDEV-18899: Server crashes in Field::set_warning_truncated_wrong_value > +--echo # > + > +set names utf8; > +set @save_optimizer_use_condition_selectivity=@@optimizer_use_condition_selectivity; > +set optimizer_use_condition_selectivity=4; > +set use_stat_tables=preferably; > +set @save_histogram_size= @@histogram_size; > +set histogram_size=255; > + > +create table t1 ( a varchar(255) character set utf8); > +insert into t1 values (REPEAT('ӥ',255)), (REPEAT('ç',255)); > + > +analyze table t1; > +select HEX(RIGHT(min_value, 1)), length(min_value) from mysql.column_stats; > +select HEX(RIGHT(max_value, 1)), length(max_value) from mysql.column_stats; > +analyze select * from t1 where a >= 'ӥ'; > + > +set @save_sql_mode= @@sql_mode; > +set sql_mode='ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION'; > +update mysql.column_stats set min_value= REPEAT('ӥ',255); > +select HEX(RIGHT(min_value, 1)), length(min_value) from mysql.column_stats; > +analyze select * from t1 where a >= 'ӥ'; > + > +set names latin1; > +drop table t1; > + > +CREATE TABLE t1 (col1 date); > +INSERT INTO t1 VALUES('2004-01-01'),('2004-02-29'); > +INSERT INTO t1 VALUES('0000-10-31'); > +analyze table t1; > +update mysql.column_stats set min_value='2004-0-31123'; > +select min_value from mysql.column_stats; > +select * from t1; > +drop table t1; > + > +set @@sql_mode= @save_sql_mode; > set use_stat_tables=@save_use_stat_tables; > +set @@histogram_size= @save_histogram_size; > +set @@optimizer_use_condition_selectivity=@save_optimizer_use_condition_selectivity; > diff --git a/sql/field.cc b/sql/field.cc > index 080cf34c76d..c26195a3650 100644 > --- a/sql/field.cc > +++ b/sql/field.cc > @@ -2219,7 +2219,6 @@ bool Field_str::can_be_substituted_to_equal_item(const Context &ctx, > return false; > } > > - > void Field_num::make_field(Send_field *field) > { > Field::make_field(field); > @@ -7023,12 +7022,16 @@ Field_longstr::check_string_copy_error(const String_copier *copier, > { > const char *pos; > char tmp[32]; > + THD *thd= get_thd(); > > if (!(pos= copier->most_important_error_pos())) > return FALSE; > > - convert_to_printable(tmp, sizeof(tmp), pos, (end - pos), cs, 6); > - set_warning_truncated_wrong_value("string", tmp); > + if (thd->count_cuted_fields) > + { > + convert_to_printable(tmp, sizeof(tmp), pos, (end - pos), cs, 6); > + set_warning_truncated_wrong_value("string", tmp); > + } > return TRUE; > } > > diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc > index b5811c683e8..0960a9a1ec5 100644 > --- a/sql/sql_statistics.cc > +++ b/sql/sql_statistics.cc > @@ -1044,6 +1044,7 @@ class Column_stat: public Stat_table > { > char buff[MAX_FIELD_WIDTH]; > String val(buff, sizeof(buff), &my_charset_bin); > + uint32 length= 0; > > for (uint i= COLUMN_STAT_MIN_VALUE; i <= COLUMN_STAT_HISTOGRAM; i++) > { > @@ -1060,7 +1061,9 @@ class Column_stat: public Stat_table > else > { > table_field->collected_stats->min_value->val_str(&val); > - stat_field->store(val.ptr(), val.length(), &my_charset_bin); > + length= Well_formed_prefix(val.charset(), val.ptr(), > + MY_MIN(val.length(), stat_field->field_length)).length(); > + stat_field->store(val.ptr(), length, &my_charset_bin); > } > break; > case COLUMN_STAT_MAX_VALUE: > @@ -1069,7 +1072,9 @@ class Column_stat: public Stat_table > else > { > table_field->collected_stats->max_value->val_str(&val); > - stat_field->store(val.ptr(), val.length(), &my_charset_bin); > + length= Well_formed_prefix(val.charset(), val.ptr(), > + MY_MIN(val.length(), stat_field->field_length)).length(); > + stat_field->store(val.ptr(), length, &my_charset_bin); > } > break; > case COLUMN_STAT_NULLS_RATIO: > @@ -2934,7 +2939,6 @@ int update_statistics_for_table(THD *thd, TABLE *table) > DBUG_RETURN(rc); > } > > - > /** > @brief > Read statistics for a table from the persistent statistical tables > @@ -2980,6 +2984,7 @@ int read_statistics_for_table(THD *thd, TABLE *table, TABLE_LIST *stat_tables) > Table_statistics *read_stats= table_share->stats_cb.table_stats; > > DBUG_ENTER("read_statistics_for_table"); > + DBUG_ASSERT(thd->count_cuted_fields == CHECK_FIELD_IGNORE); > > /* Read statistics from the statistical table table_stats */ > stat_table= stat_tables[TABLE_STAT].table; > @@ -3059,7 +3064,7 @@ int read_statistics_for_table(THD *thd, TABLE *table, TABLE_LIST *stat_tables) > } > } > } > - > + > table->stats_is_read= TRUE; > > DBUG_RETURN(0); BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

Re: [Maria-developers] 494c0448c87: MDEV-15951 system versioning by trx id doesn't work with partitioning
by Sergei Golubchik 26 Mar '19

26 Mar '19

Hi, Nikita! That's good, thanks! It's always nice to see a fix that removes more code than it adds :) But I'll reformulate the error message a bit, "Transactional system versioned tables" also means InnoDB system versioned by timestamp. I'll clarify (somehow) that it means versioned by trx_id, and push. On Mar 26, Nikita Malyavin wrote: > revision-id: 494c0448c87 (mariadb-10.3.12-65-g494c0448c87) > parent(s): 4946eb7b543 > author: Nikita Malyavin <nikitamalyavin(a)gmail.com> > committer: Nikita Malyavin <nikitamalyavin(a)gmail.com> > timestamp: 2019-02-28 19:29:12 +1000 > message: > > MDEV-15951 system versioning by trx id doesn't work with partitioning > > Fix partitioning for trx_id-versioned tables. > `partition by hash`, `range` and others now work. > `partition by system_time` is forbidden. > Currently we cannot use row_start and row_end in `partition by`, because > insertion of versioned field is done by engine's handler, as well as > row_start/row_end's value set up, which is a transaction id -- so it's > also forbidden. > > The drawback is that it's now impossible to use `partition by key()` > without parameters for such tables, because it references row_start and > row_end implicitly. > > * add handler::vers_can_native() > * drop Table_scope_and_contents_source_st::vers_native() > * drop partition_element::find_engine_flag as unused > * forbid versioning partitioning for trx_id as not supported > * adopt vers tests for trx_id partitioning > * forbid any row_end referencing in `partition by` clauses, > including implicit `by key()` > > diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt > index 9112795b754..dff12ef0a97 100644 > --- a/sql/share/errmsg-utf8.txt > +++ b/sql/share/errmsg-utf8.txt > @@ -7907,8 +7907,8 @@ ER_UNUSED_24 > ER_VERS_TEMPORARY > eng "TEMPORARY tables do not support system versioning" > > -ER_VERS_NOT_SUPPORTED > - eng "%s is not supported for %s system-versioned tables" > +ER_VERS_TRX_PART_HISTORIC_ROW_NOT_SUPPORTED > + eng "Transactional system versioned tables do not support partitioning by ROW START or ROW END" > ER_INDEX_FILE_FULL > eng "The index file for table '%-.192s' is full" Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

1 0

Re: [Maria-developers] [Commits] 842ef80b029: MDEV-18464: Port kill_one_trx fixes from 10.4 to 10.1
by Marko Mäkelä 26 Mar '19

26 Mar '19

Hi Jan, On Fri, Mar 22, 2019 at 1:59 PM jan <jan.lindstrom(a)mariadb.com> wrote: > > revision-id: 842ef80b029c975b7f7f7ec25fe55817ec626f02 (mariadb-10.1.38-68-g842ef80b029) Was this pushed to buildbot as well? I did not find the commit ID. First, some general comments: * The functions trx_start_if_not_started_low() and trx_start_for_ddl_low() differ between XtraDB and InnoDB with regard to #ifdef WITH_WSREP. Why? Are the debug assignments truly necessary? I see that the difference was introduced 5 years ago in MDEV-6247. * The files storage/{innobase,xtradb}/trx/trx0roll.cc differ from each other after the change. It looks like the innobase version is better. Please remove the unnecessary lines from xtradb: --- storage/xtradb/trx/trx0roll.cc 2019-03-26 10:20:31.767971537 +0200 +++ storage/innobase/trx/trx0roll.cc 2019-03-26 10:20:31.759971465 +0200 @@ -33,8 +33,6 @@ #include "trx0roll.ic" #endif -#include <mysql/service_wsrep.h> - #include "fsp0fsp.h" #include "mach0data.h" #include "trx0rseg.h" @@ -51,9 +49,6 @@ #include "pars0pars.h" #include "srv0mon.h" #include "trx0sys.h" -#ifdef WITH_WSREP -#include "ha_prototypes.h" -#endif /* WITH_WSREP */ /** This many pages must be undone before a truncate is tried within rollback */ @@ -1075,12 +1070,6 @@ if (trx->update_undo) { trx_undo_truncate_end(trx, trx->update_undo, limit); } - -#ifdef WITH_WSREP_OUT - if (wsrep_on(trx->mysql_thd)) { - trx->lock.was_chosen_as_deadlock_victim = FALSE; - } -#endif /* WITH_WSREP */ } /***********************************************************************//** On to the detailed review: > diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h > index fe16b8272b8..b83734b154a 100644 > --- a/storage/innobase/include/trx0trx.h > +++ b/storage/innobase/include/trx0trx.h > @@ -1,7 +1,7 @@ > /***************************************************************************** > > Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. > -Copyright (c) 2015, 2018, MariaDB Corporation. > +Copyright (c) 2015, 2019, MariaDB Corporation. > > This program is free software; you can redistribute it and/or modify it under > the terms of the GNU General Public License as published by the Free Software > @@ -613,6 +613,9 @@ struct trx_lock_t { > transaction as a victim in deadlock > resolution, it sets this to TRUE. > Protected by trx->mutex. */ > + bool was_chosen_as_replication_victim; > + /* replication has marked this > + trx to abort */ > time_t wait_started; /*!< lock wait started at this time, > protected only by lock_sys->mutex */ > > @@ -624,6 +627,12 @@ struct trx_lock_t { > only be modified by the thread that is > serving the running transaction. */ > > +#ifdef WITH_WSREP > + bool was_chosen_as_wsrep_victim; > + /*!< high priority wsrep thread has > + marked this trx to abort */ > +#endif /* WITH_WSREP */ > + > mem_heap_t* lock_heap; /*!< memory heap for trx_locks; > protected by lock_sys->mutex */ > > @@ -695,14 +704,6 @@ lock_rec_convert_impl_to_expl()) will access transactions associated > to other connections. The locks of transactions are protected by > lock_sys->mutex and sometimes by trx->mutex. */ > > -enum trx_abort_t { > - TRX_SERVER_ABORT = 0, > -#ifdef WITH_WSREP > - TRX_WSREP_ABORT, > -#endif > - TRX_REPLICATION_ABORT > -}; > - > struct trx_t{ > ulint magic_n; > > @@ -880,8 +881,6 @@ struct trx_t{ > /*------------------------------*/ > THD* mysql_thd; /*!< MySQL thread handle corresponding > to this trx, or NULL */ > - trx_abort_t abort_type; /*!< Transaction abort type*/ > - > const char* mysql_log_file_name; > /*!< if MySQL binlog is used, this field > contains a pointer to the latest file Why are you replacing trx_t::abort_type with two fields in trx_lock_t that are located far apart from each other? To avoid changing the performance too much due to caching, I would suggest to introduce the two fields in trx_t, replacing the former location of abort_type. Do we really need such long names? Would trx_t::replication_victim and trx_t::wsrep_victim suffice? Please document how concurrent access to the fields is being protected. Apparently for the WSREP field it is both trx->mutex and lock_sys->mutex. > diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc > index 3970a559a4c..984ac6cacbc 100644 > --- a/storage/innobase/lock/lock0lock.cc > +++ b/storage/innobase/lock/lock0lock.cc > @@ -4785,9 +4783,8 @@ lock_report_waiters_to_mysql( > innobase_kill_query. We mark this by setting > current_lock_mutex_owner, so we can avoid trying > to recursively take lock_sys->mutex. */ > - w_trx->abort_type = TRX_REPLICATION_ABORT; > + w_trx->lock.was_chosen_as_replication_victim = true; > thd_report_wait_for(mysql_thd, w_trx->mysql_thd); > - w_trx->abort_type = TRX_SERVER_ABORT; > } > ++i; > } Here, we seem to be only holding lock_sys->mutex, not w_trx->mutex. > @@ -7993,6 +7981,37 @@ lock_trx_handle_wait( > return DB_LOCK_WAIT; > } > > +/*********************************************************************//** > +Check whether the transaction has already been rolled back because it > +was selected as a deadlock victim, or if it has to wait then cancel > +the wait lock. > +@return DB_DEADLOCK, DB_LOCK_WAIT or DB_SUCCESS */ > +UNIV_INTERN > +dberr_t > +lock_trx_handle_wait( > +/*=================*/ > + trx_t* trx) /*!< in/out: trx lock state */ > +{ > + bool lock_mutex_taken = false; Do we really need this variable? > +#ifdef WITH_WSREP > + /* We already own mutexes */ > + if (trx->lock.was_chosen_as_wsrep_victim) { > + return lock_trx_handle_wait_low(trx); > + } > +#endif /* WITH_WSREP */ > + if (!trx->lock.was_chosen_as_replication_victim) { > + lock_mutex_enter(); > + lock_mutex_taken = true; > + } > + trx_mutex_enter(trx); > + dberr_t err = lock_trx_handle_wait_low(trx); > + if (lock_mutex_taken) { > + lock_mutex_exit(); > + } > + trx_mutex_exit(trx); > + return err; > +} I would expand the second part of the code and remove the condition before releasing the 2 mutexes: if (!trx->lock.was_chosen_as_replication_victim) { lock_mutex_enter(); trx_mutex_enter(trx); dberr_t err = lock_trx_handle_wait_low(trx); lock_mutex_exit(); trx_mutex_exit(trx); return err; } else { trx_mutex_enter(trx); dberr_t err = lock_trx_handle_wait_low(trx); trx_mutex_exit(trx); return err; } Also, I think that it would be better to declare lock_trx_handle_wait_low() only static but not inline, to reduce unnecessary code expansion. Best regards, Marko

1 0

Re: [Maria-developers] 0fa832dc426: Revert THD::THD(skip_global_sys_var_lock) argument
by Sergei Golubchik 25 Mar '19

25 Mar '19

Hi, Sergey! ok to push On Mar 25, Sergey Vojtovich wrote: > revision-id: 0fa832dc426 (mariadb-10.3.12-97-g0fa832dc426) > parent(s): 6f14489ad31 > author: Sergey Vojtovich <svoj(a)mariadb.org> > committer: Sergey Vojtovich <svoj(a)mariadb.org> > timestamp: 2019-03-21 17:17:26 +0400 > message: > > Revert THD::THD(skip_global_sys_var_lock) argument > > Originally introduced by e972125f1 to avoid harmless wait for > LOCK_global_system_variables in a newly created thread, which creation was > initiated by system variable update. > > At the same time it opens dangerous hole, when system variable update > thread already released LOCK_global_system_variables and ack_receiver > thread haven't yet completed new THD construction. In this case THD > constructor goes completely unprotected. > > Since ack_receiver.stop() waits for the thread to go down, we have to > temporarily release LOCK_global_system_variables so that it doesn't > deadlock with ack_receiver.run(). Unfortunately it breaks atomicity > of rpl_semi_sync_master_enabled updates and makes them not serialized. > > LOCK_rpl_semi_sync_master_enabled was introduced to workaround the above. > TODO: move ack_receiver start/stop into repl_semisync_master > enable_master/disable_master under LOCK_binlog protection? > > Part of MDEV-14984 - regression in connect performance > Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

1 0

Re: [Maria-developers] 6f14489ad31: Adieu find_sys_var_ex()
by Sergei Golubchik 25 Mar '19

25 Mar '19

Hi, Sergey! ok to push On Mar 25, Sergey Vojtovich wrote: > revision-id: 6f14489ad31 (mariadb-10.3.12-96-g6f14489ad31) > parent(s): 48af6851da2 > author: Sergey Vojtovich <svoj(a)mariadb.org> > committer: Sergey Vojtovich <svoj(a)mariadb.org> > timestamp: 2019-03-21 01:32:04 +0400 > message: > > Adieu find_sys_var_ex() > > Only take LOCK_plugin for plugin system variables. > > Reverted optimisation that was originally done for session tracker: it > makes much less sense now. Specifically only if connections would want to > track plugin session variables changes and these changes would actually > happen frequently. If this ever becomes an issue, there're much better > ways to optimise this workload. > > Part of MDEV-14984 - regression in connect performance > Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

1 0

Re: [Maria-developers] 48af6851da2: Make connect speed great again
by Sergei Golubchik 25 Mar '19

25 Mar '19

Hi, Sergey! On Mar 25, Sergey Vojtovich wrote: > revision-id: 48af6851da2 (mariadb-10.3.12-95-g48af6851da2) > parent(s): a83d6ee812b > author: Sergey Vojtovich <svoj(a)mariadb.org> > committer: Sergey Vojtovich <svoj(a)mariadb.org> > timestamp: 2019-03-20 01:58:42 +0400 > message: > > Make connect speed great again > > Rather than parsing session_track_system_variables when thread starts, do > it when first trackable event occurs. > > Benchmarked on a 2socket/20core/40threads Broadwell system using sysbench > connect brencmark @40 threads (with select 1 disabled): > 101379.77 -> 143016.68 CPS, whereas 10.2 is currently at 137766.31 CPS. > > Part of MDEV-14984 - regression in connect performance > > diff --git a/sql/session_tracker.cc b/sql/session_tracker.cc > index 5d98e58046d..c493c790596 100644 > --- a/sql/session_tracker.cc > +++ b/sql/session_tracker.cc > @@ -378,16 +378,9 @@ bool Session_sysvars_tracker::configure() > > bool Session_sysvars_tracker::enable(THD *thd) > { > - LEX_STRING tmp= { session_track_system_variables, > - safe_strlen(session_track_system_variables) }; > orig_list.reinit(); > - if (orig_list.parse_var_list(thd, tmp, true, thd->charset()) == true) > - { > - orig_list.reinit(); > - m_enabled= false; > - return true; > - } > - m_enabled= true; > + m_parsed= false; > + m_enabled= session_track_system_variables && *session_track_system_variables; wouldn't it be simpler not to allow both NULL and "" as valid values for session_track_system_variables ? Either always store the empty string as NULL or as "", why would you want both? > return false; > } > > @@ -530,6 +524,19 @@ void Session_sysvars_tracker::mark_as_changed(THD *thd, > { > sysvar_node_st *node; > sys_var *svar= (sys_var *)var; > + > + if (!m_parsed) > + { > + LEX_STRING tmp= { session_track_system_variables, > + safe_strlen(session_track_system_variables) }; > + if (orig_list.parse_var_list(thd, tmp, true, thd->charset())) > + { > + orig_list.reinit(); > + return; > + } > + m_parsed= true; This is of somewhat dubious value. The default value for the @@session_track_system_variables is DEFAULT("autocommit,character_set_client,character_set_connection," "character_set_results,time_zone"), and these are variables that are typically set by connectors automatically on every new connection. Only a pure C/C will benefit from it, that's what sysbench shows. If you'd tested with java, you, probably, wouldn't get any speedup. > + } > + > /* > Check if the specified system variable is being tracked, if so > mark it as changed and also set the class's m_changed flag. > diff --git a/sql/sql_class.cc b/sql/sql_class.cc > index 7194bbd03ef..53203d15f0b 100644 > --- a/sql/sql_class.cc > +++ b/sql/sql_class.cc > @@ -7320,8 +7320,13 @@ void THD::set_last_commit_gtid(rpl_gtid &gtid) > #ifndef EMBEDDED_LIBRARY > if (changed_gtid && session_tracker.session_sysvars_tracker.is_enabled()) > { > + THD *orig_thd= current_thd; > + if (orig_thd != this) When can it be true? > + set_current_thd(this); > session_tracker.session_sysvars_tracker. > mark_as_changed(this, (LEX_CSTRING*)Sys_last_gtid_ptr); > + if (orig_thd != this) > + set_current_thd(orig_thd); > } > #endif > } > Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

1 0

Re: [Maria-developers] a83d6ee812b: Safe session_track_system_variables snapshot
by Sergei Golubchik 25 Mar '19

25 Mar '19

Hi, Sergey! Just one question. Why do you need Session_sysvars_tracker::session_track_system_variables if you have THD::system_variables::session_track_system_variables ? On Mar 25, Sergey Vojtovich wrote: > revision-id: a83d6ee812b (mariadb-10.3.12-94-ga83d6ee812b) > parent(s): 3c6f17a130b > author: Sergey Vojtovich <svoj(a)mariadb.org> > committer: Sergey Vojtovich <svoj(a)mariadb.org> > timestamp: 2019-03-19 23:40:08 +0400 > message: > > Safe session_track_system_variables snapshot > > When enabling system variables tracker, make a copy of > global_system_variables.session_track_system_variables > under LOCK_global_system_variables. This protects from concurrent variable > updates and potential freed memory access, as well as from variable > reconstruction (which was previously protected by LOCK_plugin). > > We can also use this copy as a session variable pointer, so that we don't > have to allocate memory and reconstruct it every time it is referenced. > > For this very reason we don't need buffer_length stuff anymore. > > As well as don't we need to take LOCK_plugin early in ::enable(). > Unified ::parse_var_list() to acquire LOCK_plugin unconditionally. > For no apparent reason it wasn't previously acquired for global > variable update. > > No reason to access session_sysvars_tracker via get_tracker(), so access > it directly instead. > > Part of MDEV-14984 - regression in connect performance Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

1 0

Re: [Maria-developers] 3c6f17a130b: Allocate Session_sysvars_tracker statically
by Sergei Golubchik 25 Mar '19

25 Mar '19

Hi, Sergey! Please, see few questions below On Mar 25, Sergey Vojtovich wrote: > revision-id: 3c6f17a130b (mariadb-10.3.12-93-g3c6f17a130b) > parent(s): db9854b7d48 > author: Sergey Vojtovich <svoj(a)mariadb.org> > committer: Sergey Vojtovich <svoj(a)mariadb.org> > timestamp: 2019-03-19 23:40:07 +0400 > message: > > Allocate Session_sysvars_tracker statically > > One less new/delete per connection. > > Removed m_mem_flag since most allocs are thread specific. The only > exception are allocs performed during initialization. > > Removed State_tracker and Session_tracker constructors as they don't make > sense anymore. > > Rather than having session_tracker.deinit(), fixed memory accounting so > that it gets initialized before any other constructor and deinitialized > after all destructors done. Cons: makes THD a few pointers larger :( > > Part of MDEV-14984 - regression in connect performance > > diff --git a/sql/sql_class.h b/sql/sql_class.h > index 69e6f58f854..64d772e5d17 100644 > --- a/sql/sql_class.h > +++ b/sql/sql_class.h > @@ -2145,13 +2145,34 @@ struct wait_for_commit > > extern "C" void my_message_sql(uint error, const char *str, myf MyFlags); > > + > +/** > + A wrapper around thread_count. > + > + It must be specified as a first base class of THD, so that increment is > + done before any other THD constructors and decrement - after any other THD > + destructors. > + > + Destructor unblocks close_conneciton() if there are no more THD's left. > +*/ > +class THD_count > +{ > + THD *orig_thd; > + THD_count(); > + ~THD_count(); > + friend class THD; > +}; 1. Why a separate class, what's wrong with having it in THD? No downcasts, no risk that someone would try to use it separately. 2. orig_thd looks very suspicious. You preserve the old THD* pointer for the whole duration of this THD lifetime and restore it at the end. How can you know that the "orig_thd" is still valid? Why do you need to do it anyway? > + > + > /** > @class THD > For each client connection we create a separate thread with THD serving as > a thread/connection descriptor > */ > > -class THD :public Statement, > +class THD: public THD_count, /* this must be first */ an assert would be better than "/* must be first */". Like, assert(&orig_thd == this) or something. > + public Statement, > + > /* > This is to track items changed during execution of a prepared > statement/stored procedure. It's created by > @@ -2176,19 +2197,6 @@ class THD :public Statement, > inline bool is_conventional() const > { DBUG_ASSERT(0); return Statement::is_conventional(); } > > - void dec_thread_count(void) > - { > - DBUG_ASSERT(thread_count > 0); > - thread_safe_decrement32(&thread_count); > - signal_thd_deleted(); > - } > - > - > - void inc_thread_count(void) > - { > - thread_safe_increment32(&thread_count); > - } > - > public: > MDL_context mdl_context; > > diff --git a/sql/sql_class.cc b/sql/sql_class.cc > index 9c9fa228e2a..afe4123a52c 100644 > --- a/sql/sql_class.cc > +++ b/sql/sql_class.cc > @@ -597,8 +597,46 @@ extern "C" void thd_kill_timeout(THD* thd) > thd->awake(KILL_TIMEOUT); > } > > -THD::THD(my_thread_id id, bool is_wsrep_applier, bool skip_global_sys_var_lock) > - :Statement(&main_lex, &main_mem_root, STMT_CONVENTIONAL_EXECUTION, > + > +THD_count::THD_count() > +{ > + THD *thd= static_cast<THD*>(this); > + thread_safe_increment32(&thread_count); > + /* > + We set THR_THD to temporally point to this THD to register all the > + variables that allocates memory for this THD > + */ > + orig_thd= current_thd; > + set_current_thd(thd); > + thd->status_var.local_memory_used= sizeof(THD); > + thd->status_var.max_local_memory_used= thd->status_var.local_memory_used; > + thd->status_var.global_memory_used= 0; > + thd->variables.max_mem_used= global_system_variables.max_mem_used; > +} > + > + > +THD_count::~THD_count() > +{ > + THD *thd= static_cast<THD*>(this); > + if (thd->status_var.local_memory_used != 0) > + { > + DBUG_PRINT("error", ("memory_used: %lld", > + thd->status_var.local_memory_used)); > + SAFEMALLOC_REPORT_MEMORY(thd->thread_id); > + DBUG_ASSERT(thd->status_var.local_memory_used == 0 || > + !debug_assert_on_not_freed_memory); > + } > + update_global_memory_status(thd->status_var.global_memory_used); > + set_current_thd(orig_thd); > + DBUG_ASSERT(thread_count > 0); > + thread_safe_decrement32(&thread_count); > + signal_thd_deleted(); > +} > + > + > +THD::THD(my_thread_id id, bool is_wsrep_applier, bool skip_global_sys_var_lock): > + THD_count(), > + Statement(&main_lex, &main_mem_root, STMT_CONVENTIONAL_EXECUTION, > /* statement id */ 0), > rli_fake(0), rgi_fake(0), rgi_slave(NULL), > protocol_text(this), protocol_binary(this), > @@ -654,15 +692,6 @@ THD::THD(my_thread_id id, bool is_wsrep_applier, bool skip_global_sys_var_lock) > ulong tmp; > bzero(&variables, sizeof(variables)); > > - /* > - We set THR_THD to temporally point to this THD to register all the > - variables that allocates memory for this THD > - */ > - THD *old_THR_THD= current_thd; > - set_current_thd(this); > - status_var.local_memory_used= sizeof(THD); > - status_var.max_local_memory_used= status_var.local_memory_used; > - status_var.global_memory_used= 0; > variables.pseudo_thread_id= thread_id; > variables.max_mem_used= global_system_variables.max_mem_used; > main_da.init(); > @@ -849,8 +878,7 @@ THD::THD(my_thread_id id, bool is_wsrep_applier, bool skip_global_sys_var_lock) > save_prep_leaf_list= FALSE; > org_charset= 0; > /* Restore THR_THD */ > - set_current_thd(old_THR_THD); > - inc_thread_count(); > + set_current_thd(orig_thd); > } > > > @@ -1589,7 +1617,6 @@ void THD::reset_for_reuse() > > THD::~THD() > { > - THD *orig_thd= current_thd; > THD_CHECK_SENTRY(this); > DBUG_ENTER("~THD()"); > /* Check that we have already called thd->unlink() */ > @@ -1601,7 +1628,11 @@ THD::~THD() > In error cases, thd may not be current thd. We have to fix this so > that memory allocation counting is done correctly > */ > - set_current_thd(this); > + orig_thd= current_thd; > + if (orig_thd == this) > + orig_thd= 0; > + else > + set_current_thd(this); > if (!status_in_global) > add_status_to_global(); > > @@ -1655,22 +1686,6 @@ THD::~THD() > lf_hash_put_pins(xid_hash_pins); > /* Ensure everything is freed */ > status_var.local_memory_used-= sizeof(THD); > - > - /* trick to make happy memory accounting system */ > -#ifndef EMBEDDED_LIBRARY > - session_tracker.deinit(); > -#endif //EMBEDDED_LIBRARY > - > - if (status_var.local_memory_used != 0) > - { > - DBUG_PRINT("error", ("memory_used: %lld", status_var.local_memory_used)); > - SAFEMALLOC_REPORT_MEMORY(thread_id); > - DBUG_ASSERT(status_var.local_memory_used == 0 || > - !debug_assert_on_not_freed_memory); > - } > - update_global_memory_status(status_var.global_memory_used); > - set_current_thd(orig_thd == this ? 0 : orig_thd); > - dec_thread_count(); > DBUG_VOID_RETURN; > } Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

1 0