- developers - lists.mariadb.org

Re: [Maria-developers] 4c263b6d30b: MDEV-20632: Recursive CTE cycle detection using CYCLE clause
by Sergei Golubchik 26 Feb '20

26 Feb '20

Hi, Sanja! _some_ comments are below. The main thing, I still don't understand your changes in sql_select.cc. Why did you create separate counters and code paths with if() for CYCLE? I'd think it could be just a generalization of DISTINCT On Feb 20, Oleksandr Byelkin wrote: > revision-id: 4c263b6d30b (mariadb-10.5.0-92-g4c263b6d30b) > parent(s): 6f2e2285291 > author: Oleksandr Byelkin <sanja(a)mariadb.com> > committer: Oleksandr Byelkin <sanja(a)mariadb.com> > timestamp: 2020-01-27 21:56:02 +0100 > message: > > MDEV-20632: Recursive CTE cycle detection using CYCLE clause > > Added CYCLE clause to recursive CTE. > > diff --git a/sql/item.h b/sql/item.h > index c1f68a4f942..2cbaf880e00 100644 > --- a/sql/item.h > +++ b/sql/item.h > @@ -622,6 +622,13 @@ class st_select_lex_unit; > class Item_func_not; > class Item_splocal; > > +/* Item::common_flags */ > +/* Indicates that name of this Item autogenerated or set by user */ > +#define IS_AUTO_GENETATED_NAME 1 typo ^^^ > +/* Inticates that this item is in CYCLE clause of WITH */ typo ^^^ > +#define IS_IN_WITH_CYCLE 2 > + > + > /** > String_copier that sends Item specific warnings. > */ > diff --git a/sql/sql_cte.cc b/sql/sql_cte.cc > index 5b3d3108da5..971844120bc 100644 > --- a/sql/sql_cte.cc > +++ b/sql/sql_cte.cc > @@ -982,6 +982,38 @@ With_element::rename_columns_of_derived_unit(THD *thd, are you sure what you're doing below can still be called "rename_columns_of_derived_unit" ? > else > make_valid_column_names(thd, select->item_list); > > + if (cycle_list) > + { > + List_iterator_fast<Item> it(select->item_list); > + List_iterator_fast<LEX_CSTRING> nm(*cycle_list); > + List_iterator_fast<LEX_CSTRING> nm_check(*cycle_list); > + DBUG_ASSERT(cycle_list->elements != 0); > + while (LEX_CSTRING *name= nm++) > + { > + Item *item; > + // unique check > + LEX_CSTRING *check; > + nm_check.rewind(); > + while ((check= nm_check++) && check != name) > + { > + if (check->length == name->length && > + strncmp(check->str, name->str, name->length) == 0) > + { > + my_error(ER_DUP_FIELDNAME, MYF(0), check->str); > + return true; > + } > + } > + while ((item= it++) && > + (item->name.length != name->length || > + strncmp(item->name.str, name->str, name->length) != 0)); > + if (item == NULL) > + { > + my_error(ER_BAD_FIELD_ERROR, MYF(0), name->str, "CYCLE clause"); > + return true; > + } this pattern is used so often that we might want to have a helper, like, List_iterator_fast<>::find() not in this commit, though. > + item->common_flags|= IS_IN_WITH_CYCLE; > + } > + } > unit->columns_are_renamed= true; > > return false; > @@ -1425,6 +1457,21 @@ void With_clause::print(String *str, enum_query_type query_type) > } > > > +static void list_strlex_print(String *str, List<LEX_CSTRING> *list) > +{ > + List_iterator_fast<LEX_CSTRING> li(*list); > + bool first= TRUE; > + while(LEX_CSTRING *col_name= li++) > + { > + if (first) > + first= FALSE; > + else > + str->append(','); > + str->append(col_name); should be "append_identifier", shouldn't it? add a test where a column name is a reserved word. > + } > +} > + > + > /** > @brief > Print this with element > @@ -1444,22 +1491,20 @@ void With_element::print(String *str, enum_query_type query_type) > { > List_iterator_fast<LEX_CSTRING> li(column_list); > str->append('('); > - for (LEX_CSTRING *col_name= li++; ; ) > - { > - str->append(col_name); > - col_name= li++; > - if (!col_name) > - { > - str->append(')'); > - break; > - } > - str->append(','); > - } > + list_strlex_print(str, &column_list); any other code that could make use of your list_strlex_print ? > + str->append(')'); > } > - str->append(STRING_WITH_LEN(" as ")); > - str->append('('); > + str->append(STRING_WITH_LEN(" as (")); > spec->print(str, query_type); > str->append(')'); > + > + if (cycle_list) > + { > + DBUG_ASSERT(cycle_list->elements != 0); > + str->append(STRING_WITH_LEN(" CYCLE (")); > + list_strlex_print(str, cycle_list); > + str->append(") "); > + } > } > > > diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy > index c115d9352aa..0d60ab8579f 100644 > --- a/sql/sql_yacc.yy > +++ b/sql/sql_yacc.yy > @@ -14704,9 +14704,30 @@ with_list_element: > if (elem->set_unparsed_spec(thd, spec_start, $6.pos(), > spec_start - query_start)) > MYSQL_YYABORT; > + if ($7) > + { > + elem->set_cycle_list($7); > + } > } > ; > > +opt_cycle: > + /* empty */ > + { $$= NULL; } > + | > + CYCLE_SYM > + { > + if (!Lex->curr_with_clause->with_recursive) > + { > + thd->parse_error(ER_SYNTAX_ERROR, $1.pos()); > + } > + } > + '(' with_column_list ')' Where did you see that the standard requires parentheses here? > + { > + $$= $4; > + } > + ; > + > > opt_with_column_list: > /* empty */ Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

[Maria-developers] Re MDEV-21610, cleanup patch
by Sergey Petrunia 24 Feb '20

24 Feb '20

Hello Igor, > @spetrunia: your "cleanup" of commit e637355156cb28388a291b0e3a5e9ee863b2854d > actually changes the architecture: you forced the pushed filter to be the > same as the filter of SQL layer. I'm not sure I understand that. I assume the layers are: 1. SQL layer 2. handler level (generic, with ha_xxx methods) 3. DS-MRR layer 4. The storage engine internals handler::pushed_rowid_filter is at level 2. Its value matches the rowid filter that has been pushed to level 4. What the code does after my fix: // Take the filter from level 2, put it to level 3 (DS-MRR) rowid_filter= h_arg->pushed_rowid_filter; // Inform level 4 that it doesn't need to do filtering // (This also affects level 2 as handler::pushed_rowid_filter will be // cleared) h_arg->cancel_pushed_rowid_filter(); I see a slight problem here that handler->pushed_rowid_filter will be NULL while actually storage engine + MRR (levels 2,3,4 together) will still be checking the rowid filter. But this seems to be unavoidable in current design (and it was also the case before my cleanup patch). > Although they are the same now for each engine where filters are allowed > this might be not the case in future development. I'm not sure what scenario you have in mind, could you please elaborate? > I agree that accessing filter from the handler layer is not a clean solution, > but in this case the owner of SQL layer filter must be the TABLE structure. > Summing up: if we decide not to use a pushed filter in the engine we have to > take the filter from SQL layer. So, you're suggesting that taking the filter from handler::pushed_rowid_filter is worse than taking it from table->reginfo.join_tab ? BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

Re: [Maria-developers] MDEV-21580: Allow packed sort keys in sort buffer
by Sergey Petrunia 20 Feb '20

20 Feb '20

Hi Varun, Please find some input below. > commit f75829eebe96db55508cbc03c967e1c340da0cfc > Author: Varun Gupta <varun.gupta(a)mariadb.com> > Date: Fri Feb 7 02:30:06 2020 +0530 > > MDEV-21580: Allow packed sort keys in sort buffer > > This task deals with packing the sort key inside the sort buffer, which would > lead to efficient usage of the memory allocated for the sort buffer. > > The changes brought by this feature are > 1) Sort buffers would have sort keys of variable length > 2) The format for sort keys inside the sort buffer would look like > |<sort_length><null_byte><key_part1><null_byte><key_part2>.......| > sort_length is the extra bytes that are required to store the variable > length of a sort key. > 3) When packing of sort key is done we store the ORIGINAL VALUES inside > the sort buffer and not the STRXFRM form (mem-comparable sort keys). > 4) Special comparison function packed_keys_comparison() is introduced > to compare 2 sort keys. > > diff --git a/mysql-test/main/mdev21580.result b/mysql-test/main/mdev21580.result > new file mode 100644 > index 00000000000..c504b79d52f > --- /dev/null > +++ b/mysql-test/main/mdev21580.result > @@ -0,0 +1,6427 @@ > +create table t1(a int); > diff --git a/mysql-test/main/mdev21580.test b/mysql-test/main/mdev21580.test This test is too long, I assume it will be gone after the "diff solution" is implemented. > diff --git a/sql/filesort.cc b/sql/filesort.cc > index 763f9f59246..1cdc8e7af00 100644 > --- a/sql/filesort.cc > +++ b/sql/filesort.cc > @@ -215,16 +219,14 @@ SORT_INFO *filesort(THD *thd, TABLE *table, Filesort *filesort, > error= 1; > sort->found_rows= HA_POS_ERROR; > > - param.init_for_filesort(sortlength(thd, filesort->sortorder, s_length, > - &multi_byte_charset), > - table, max_rows, filesort->sort_positions); > + param.sort_keys= sort_keys; > + uint sort_len= sortlength(thd, sort_keys, &multi_byte_charset, > + &allow_packing_for_sortkeys); psergey: I think this doesn't work as intended. Consider the two tests: create table t2 (a text collate utf8_unicode_ci, b varchar(32)); insert into t2 select a,a from ten; Q1: select * from t2 order by a; Q2: select * from t2 order by a, b; When debugging Q1, I can see allow_packing_for_sortkeys=false after the sortlength() call. This is ok I think. When debugging Q2, can see allow_packing_for_sortkeys= true after the sortlength() call. This is wrong. > @@ -491,12 +503,20 @@ uint Filesort::make_sortorder(THD *thd, JOIN *join, table_map first_table_bit) > for (ord = order; ord; ord= ord->next) > count++; > if (!sortorder) > - sortorder= (SORT_FIELD*) thd->alloc(sizeof(SORT_FIELD) * (count + 1)); > + sortorder= (SORT_FIELD*) thd->alloc(sizeof(SORT_FIELD) * count); > + void *rawmem= thd->alloc(sizeof(Sort_keys)); > pos= sort= sortorder; > > if (!pos) > DBUG_RETURN(0); > > + Sort_keys_array sort_keys_array(sortorder, count); > + Sort_keys* sort_keys= new (rawmem) Sort_keys(sort_keys_array); > + psergey: Why not inherit the class from Sql_alloc and use its operator new? The above is probably correct but it makes me wonder what is it doing every time I encounter this piece of code. > + if (!sort_keys) > + DBUG_RETURN(0); psergey: You're checking this (which cant fail) but not checking the result of thd->alloc() call above (which can fail)? > + > + pos= sort_keys->begin(); > for (ord= order; ord; ord= ord->next, pos++) > { > Item *first= ord->item[0]; ... > +/* > + @brief > + Comparison function to compare two packed sort keys > + > + @param sort_param cmp argument > + @param a_ptr packed sort key > + @param b_ptr packed sort key > + > + @retval > + >0 key a_ptr greater than b_ptr > + =0 key a_ptr equal to b_ptr > + <0 key a_ptr less than b_ptr > + > +*/ > + psergey: function names typically start with a verb. Can this one follow the convention? > +int packed_keys_comparison(void *sort_param, > + unsigned char **a_ptr, unsigned char **b_ptr) > +{ > + int retval= 0; > + size_t a_len, b_len; > + Sort_param *param= (Sort_param*)sort_param; > + Sort_keys *sort_keys= param->sort_keys; > + uchar *a= *a_ptr; > + uchar *b= *b_ptr; > + ... > @@ -772,6 +777,21 @@ static int rr_cmp(uchar *a,uchar *b) > #endif > } > > + > +/** > + Copy (unpack) values appended to sorted fields from a buffer back to > + their regular positions specified by the Field::ptr pointers. > + > + @param addon_field Array of descriptors for appended fields > + @param buff Buffer which to unpack the value from > + > + @note > + The function is supposed to be used only as a callback function > + when getting field values for the sorted result set. > + > + @return > + void. psergey: the above two lines do not have any value, please remove :) > +*/ > template<bool Packed_addon_fields> > inline void SORT_INFO::unpack_addon_fields(uchar *buff) > { BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 1

Re: [Maria-developers] 8790c095ab0: MDEV-21704 Add a new JSON field "version_id" into mysql.global_priv.priv
by Sergei Golubchik 20 Feb '20

20 Feb '20

Hi, Alexander! Looks ok, just one test related comment. On Feb 20, Alexander Barkov wrote: > revision-id: 8790c095ab0 (mariadb-10.5.0-225-g8790c095ab0) > parent(s): 2c34315df6e > author: Alexander Barkov <bar(a)mariadb.com> > committer: Alexander Barkov <bar(a)mariadb.com> > timestamp: 2020-02-14 21:56:12 +0400 > message: > > MDEV-21704 Add a new JSON field "version_id" into mysql.global_priv.priv > > +--log-error=$MYSQLTEST_VARDIR/tmp/system_mysql_db_error_log.err ... > +CREATE TABLE error_log (msg TEXT); > +--disable_query_log > +--eval LOAD DATA INFILE "$MYSQLTEST_VARDIR/tmp/system_mysql_db_error_log.err" INTO TABLE error_log; > +--enable_query_log > + > +# On some platforms hours less than 10 can be logged without the leading zero: > +# '2020-02-13 8:16:56' rather than > +# '2020-02-13 08:16:56' > +# Hence the space character in the hour part of the regular expression. > +# Also, replace '\r' to '', so the output is equal on Linux and Windows. > + > +SELECT > + REPLACE( > + REGEXP_REPLACE(msg, > + '^[0-9]{4}-[0-9]{2}-[0-9]{2} [ 0-9]{2}:[0-9]{2}:[0-9]{2}', > + 'YYYY-MM-DD hh:mm:ss'), > + '\r','' > + ) AS msg > +FROM error_log WHERE msg LIKE '%user%entry%'; instead of --log-error and LOAD DATA with REGEXP_REPLACE, please use something like --let SEARCH_FILE=$MYSQLTEST_VARDIR/log/mysqld.1.err --let SEARCH_PATTERN="'user' entry 'bad_.*has a wrong" --source include/search_pattern_in_file.inc this will print that there were 4 matches. Alternatively you can use four very specific SEARCH_PATTERN, one for each expected error. It's easier than what you've used, keeps the error log in one file, and doesn't run unnecessary sql on the server (otherwise debugging quickly becomes very annoying, grr, replication tests). Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] bd98ef106c8: MDEV-17554 Auto-create new partition for system versioned tables with history partitioned by INTERVAL/LIMIT
by Sergei Golubchik 19 Feb '20

19 Feb '20

Hi, Aleksey! 1. It should be possible to enable/disable auto-creation. For example, CREATE TABLE ... PARTITION BY SYSTEM_TIME ... PARTITIONS AUTO; this solves few problems at once: * a user explicitly tells when auto-creation should work * you don't need to worry "if this name is already occupied" * you can define that these partitions can never overflow (for INTERVAL) * if you know that AUTO partitions never overflow, you can keep the old behavior for ALTER TABLE ADD PARTITION. 2. Separate thread is an interesting solution. Nicely avoids lots of problems. Still: * it's asynchronous, all tests have to take it into account * it's racy. "low probability" or not, with our number of users they'll hit it and rightfully will report it as a bug * if LOCK TABLE prevents it, partitions can overflow But I think these problems are easier to solve than those we'll face if auto-creation will happen in the main connection thread. So, let's keep your approach with a thread. But instead of going through the parser and mysql_execute_command, create a function that takes a TABLE or a TABLE_SHARE. And adds a partition there. It'll fix the "racy" part. This function can be called from a new thread. As for the LOCK TABLES - if you're under LOCK TABLES, you can simply call that function (to add a partition) directly at the end of the main INSERT/UPDATE/DELETE statement. It'll solve the last problem, LOCK TABLES won't prevent auto-creation. A couple of style comments: 1. create a one-liner make_partition_name (or something) with: sprintf(move_ptr, "p%u", i) and use it both in create_default_partition_names() and in your new code. Just to make sure partition names are always generated uniformly by the same function and if we'll want to change it to "pp%u" it should be done only in one place. 2. don't overload operators, use, say, bool greater_than(size_t seconds) But overall it's pretty good. A nice idea with a separate thread. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org On Nov 07, Aleksey Midenkov wrote: > revision-id: bd98ef106c8 (mariadb-10.4.7-55-gbd98ef106c8) > parent(s): 6d1fa01e8f1 > author: Aleksey Midenkov <midenok(a)gmail.com> > committer: Aleksey Midenkov <midenok(a)gmail.com> > timestamp: 2019-08-19 12:01:14 +0300 > message: > > MDEV-17554 Auto-create new partition for system versioned tables with history partitioned by INTERVAL/LIMIT > > When there are E empty partitions left, auto-create N new empty > partitions. > > This scheme must not allow partition overflow. I.e. E-fill time must > not exceed N-creation time. This means that low values for INTERVAL > and LIMIT must not be allowed for auto-creation. In case when overflow > is detected there is no need to do anything special: a warning will be > issued and the user will run manual rebuild to redistribute records > correctly. This is important because automatic ADD must be done fast, > without forced rebuild, by the obvious reason. > > Initial version implements hard-coded values of 1 for E and N. As well > as auto-creation threshold MinInterval = 1 hour, MinLimit = 1000. > > The name for newly added partition will be first chosen as "pX", where > X is partition number and "p" is hard-coded name prefix. If this name > is already occupied, the X will be incremented until the resulting > name will be free to use. > > Auto-creation mechanism is applied to every table having LIMIT or > INTERVAL clause. Note that there is no much sense in specifying > explicit partition list in this case and this is covered by > MDEV-19903. The syntax to explicitly turn it on/off as well as > user-defined values for E, N and name prefix is subject for further > discussion and feature requests. > > ALTER TABLE ADD PARTITION is now always fast. If there some history > partition overflow occurs manual ALTER TABLE REBUILD PARTITION is > needed.

2 5

[Maria-developers] Fixing 10.4+ build on OpenBSD
by Brad Smith 15 Feb '20

15 Feb '20

Hi, Looking for some assistance in getting a patch commited to resolve MDEV-19603. Since Galera was upgraded from 3 to 4 the CMake bits include a hardcoded libdl which broke the build of 10.4+ on OpenBSD.

1 0

Re: [Maria-developers] [Commits] a906aae: MDEV-21610 Different query results from 10.4.11 to 10.4.12
by Sergey Petrunia 14 Feb '20

14 Feb '20

Hello Igor, I've found a deficiency in the scheme used by the patch: it doesn't work if the join buffer is re-filled multiple times. On the first execution, everything works as intended. dsmrr_init() executes these lines: rowid_filter= h_arg->pushed_rowid_filter; h_arg->cancel_pushed_rowid_filter(); then I can see that Mrr_ordered_rndpos_reader::refill_from_index_reader uses the filter. The MRR scan continues until it finishes. Then, SQL layer fills the join buffer again, and calls multi_range_read_init() again, which calls dsmrr_init(). And here, h_arg->pushed_rowid_filter==NULL (as we've cleared it previously), and the second MRR scan is not used anymore. Example that I used for debugging (maybe it's larger than necessary): create table t10 ( pk int primary key, a int, b int, filler char(100), key(a), key(b) ); insert into t10 select A.a + 1000 *B.a, A.a + 1000 *B.a, A.a + 1000 *B.a, 'filler-data=FILLER=DATA' from one_k A, one_k B; create table t11 (a int); insert into t11 select a from one_k where a < 200; set optimizer_switch='mrr=on'; set join_cache_level=6; set join_buffer_size=128; MariaDB [test]> explain select * from t11, t10 where t10.a=t11.a and t10.b < 300; +------+-------------+-------+------------+---------------+------+---------+------------+--------+-----------------------------------------------------------------------------------------+ | id | select_type | table | type | possible_keys | key | key_len | ref | rows | Extra | +------+-------------+-------+------------+---------------+------+---------+------------+--------+-----------------------------------------------------------------------------------------+ | 1 | SIMPLE | t11 | ALL | NULL | NULL | NULL | NULL | 200 | Using where | | 1 | SIMPLE | t10 | ref|filter | a,b | a|b | 5|5 | test.t11.a | 1 (0%) | Using where; Using join buffer (flat, BKA join); Rowid-ordered scan; Using rowid filter | +------+-------------+-------+------------+---------------+------+---------+------------+--------+-----------------------------------------------------------------------------------------+ select * from t11, t10 where t10.a=t11.a and t10.b < 300; Ideas about the solution: 1. DS-MRR code should put the rowid filter back into the 'h_arg' handler when the scan finishes (i.e. returns HA_ERR_END_OF_FILE) 2. DS-MRR code should store the rowid filter internally (and clear it up when dsmrr_close() is called. I haven't investigated which is better. On Thu, Feb 13, 2020 at 10:55:56PM -0800, IgorBabaev wrote: > revision-id: a906aaee26a7be57fe2db62214179476ec124486 (mariadb-10.4.11-38-ga906aae) > parent(s): 7ea413ac2d80c7f03d1dbad90ac30ecddd8b2835 > author: Igor Babaev > committer: Igor Babaev > timestamp: 2020-02-13 22:55:56 -0800 > message: > > MDEV-21610 Different query results from 10.4.11 to 10.4.12 > > This patch fixes the following defects/bugs. > 1. If BKA[H] algorithm was used to join a table for which the optimizer > had decided to employ a rowid filter the filter actually was not built. > 2. The patch for the bug MDEV-21356 that added the code canceling pushing > rowid filter into an engine for the table joined with employment of > BKA[H] and MRR was not quite correct for Innodb engine because this > cancellation was done after InnoDB code had already bound the the pushed > filter to internal InnoDB structures. > > --- > mysql-test/main/rowid_filter_innodb.result | 333 +++++++++++++++++++++++++++++ > mysql-test/main/rowid_filter_innodb.test | 153 +++++++++++++ > sql/multi_range_read.cc | 39 ++-- > sql/multi_range_read.h | 5 +- > sql/opt_range.cc | 3 +- > sql/sql_join_cache.cc | 2 + > 6 files changed, 515 insertions(+), 20 deletions(-) > > diff --git a/mysql-test/main/rowid_filter_innodb.result b/mysql-test/main/rowid_filter_innodb.result > index c59b95b..9423fb1 100644 > --- a/mysql-test/main/rowid_filter_innodb.result > +++ b/mysql-test/main/rowid_filter_innodb.result > @@ -2522,3 +2522,336 @@ id select_type table type possible_keys key key_len ref rows r_rows filtered r_f > 1 SIMPLE t1 index a,b PRIMARY 4 NULL 3008 3008.00 1.36 0.00 Using where > DROP TABLE t1; > SET global innodb_stats_persistent= @stats.save; > +# > +# MDEV-21610: Using rowid filter with BKA+MRR > +# > +set @stats.save= @@innodb_stats_persistent; > +set global innodb_stats_persistent=on; > +CREATE TABLE acli ( > +id bigint(20) NOT NULL, > +rid varchar(255) NOT NULL, > +tp smallint(6) NOT NULL DEFAULT 0, > +PRIMARY KEY (id), > +KEY acli_rid (rid), > +KEY acli_tp (tp) > +) ENGINE=InnoDB DEFAULT CHARSET=utf8; > +insert into acli(id,rid,tp) values > +(184929059698905997,'ABABABABABABABABAB',103), > +(184929059698905998,'ABABABABABABABABAB',121), > +(283586039035985921,'00000000000000000000000000000000',103), > +(2216474704108064678,'020BED6D07B741CE9B10AB2200FEF1DF',103), > +(2216474704108064679,'020BED6D07B741CE9B10AB2200FEF1DF',121), > +(3080602882609775593,'B5FCC8C7111E4E3CBC21AAF5012F59C2',103), > +(3080602882609775594,'B5FCC8C7111E4E3CBC21AAF5012F59C2',121), > +(3080602882609776594,'B5FCC8C7111E4E3CBC21AAF5012F59C2',121), > +(3080602882609777595,'B5FCC8C7111E4E3CBC21AAF5012F59C2',121), > +(4269412446747236214,'SCSCSCSCSCSCSCSC',103), > +(4269412446747236215,'SCSCSCSCSCSCSCSC',121), > +(6341490487802728356,'6072D47E513F4A4794BBAB2200FDB67D',103), > +(6341490487802728357,'6072D47E513F4A4794BBAB2200FDB67D',121); > +CREATE TABLE acei ( > +id bigint(20) NOT NULL, > +aclid bigint(20) NOT NULL DEFAULT 0, > +atp smallint(6) NOT NULL DEFAULT 0, > +clus smallint(6) NOT NULL DEFAULT 0, > +PRIMARY KEY (id), > +KEY acei_aclid (aclid), > +KEY acei_clus (clus) > +) ENGINE=InnoDB DEFAULT CHARSET=utf8; > +insert into acei(id,aclid,atp,clus) values > +(184929059698905999,184929059698905997,0,1), > +(184929059698906000,184929059698905997,0,1), > +(184929059698906001,184929059698905997,1,1), > +(184929059698906002,184929059698905998,1,1), > +(283586039035985922,283586039035985921,1,1), > +(2216474704108064684,2216474704108064678,0,1), > +(2216474704108064685,2216474704108064678,0,1), > +(2216474704108064686,2216474704108064678,1,1), > +(2216474704108064687,2216474704108064679,1,1), > +(3080602882609775595,3080602882609775593,0,1), > +(3080602882609775596,3080602882609775593,0,1), > +(3080602882609775597,3080602882609775593,1,1), > +(3080602882609775598,3080602882609775594,1,1), > +(3080602882609776595,3080602882609776594,1,1), > +(3080602882609777596,3080602882609777595,1,1), > +(4269412446747236216,4269412446747236214,0,1), > +(4269412446747236217,4269412446747236214,0,1), > +(4269412446747236218,4269412446747236214,1,1), > +(4269412446747236219,4269412446747236215,1,1), > +(6341490487802728358,6341490487802728356,0,1), > +(6341490487802728359,6341490487802728356,0,1), > +(6341490487802728360,6341490487802728356,1,1), > +(6341490487802728361,6341490487802728357,1,1); > +CREATE TABLE filt ( > +id bigint(20) NOT NULL, > +aceid bigint(20) NOT NULL DEFAULT 0, > +clid smallint(6) NOT NULL DEFAULT 0, > +fh bigint(20) NOT NULL DEFAULT 0, > +PRIMARY KEY (id), > +KEY filt_aceid (aceid), > +KEY filt_clid (clid), > +KEY filt_fh (fh) > +) ENGINE=InnoDB DEFAULT CHARSET=utf8; > +insert into filt(id,aceid,clid,fh) values > +(184929059698905999,184929059698905999,1,8948400944397203540), > +(184929059698906000,184929059698906000,1,-3516039679025944536), > +(184929059698906001,184929059698906001,1,-3516039679025944536), > +(184929059698906002,184929059698906001,1,2965370193075218252), > +(184929059698906003,184929059698906001,1,8948400944397203540), > +(184929059698906004,184929059698906002,1,2478709353550777738), > +(283586039035985922,283586039035985922,1,5902600816362013271), > +(2216474704108064686,2216474704108064684,1,8948400944397203540), > +(2216474704108064687,2216474704108064685,1,-7244708939311117030), > +(2216474704108064688,2216474704108064686,1,-7244708939311117030), > +(2216474704108064689,2216474704108064686,1,7489060986210282479), > +(2216474704108064690,2216474704108064686,1,8948400944397203540), > +(2216474704108064691,2216474704108064687,1,-3575268945274980038), > +(3080602882609775595,3080602882609775595,1,8948400944397203540), > +(3080602882609775596,3080602882609775596,1,-5420422472375069774), > +(3080602882609775597,3080602882609775597,1,-5420422472375069774), > +(3080602882609775598,3080602882609775597,1,8518228073041491534), > +(3080602882609775599,3080602882609775597,1,8948400944397203540), > +(3080602882609775600,3080602882609775598,1,6311439873746261694), > +(3080602882609775601,3080602882609775598,1,6311439873746261694), > +(3080602882609776595,3080602882609776595,1,-661101805245999843), > +(3080602882609777596,3080602882609777596,1,-661101805245999843), > +(3080602882609777597,3080602882609777596,1,2216865386202464067), > +(4269412446747236216,4269412446747236216,1,8948400944397203540), > +(4269412446747236217,4269412446747236217,1,-1143096194892676000), > +(4269412446747236218,4269412446747236218,1,-1143096194892676000), > +(4269412446747236219,4269412446747236218,1,5313391811364818290), > +(4269412446747236220,4269412446747236218,1,8948400944397203540), > +(4269412446747236221,4269412446747236219,1,7624499822621753835), > +(6341490487802728358,6341490487802728358,1,8948400944397203540), > +(6341490487802728359,6341490487802728359,1,8141092449587136068), > +(6341490487802728360,6341490487802728360,1,8141092449587136068), > +(6341490487802728361,6341490487802728360,1,1291319099896431785), > +(6341490487802728362,6341490487802728360,1,8948400944397203540), > +(6341490487802728363,6341490487802728361,1,6701841652906431497); > +analyze table filt, acei, acli; > +Table Op Msg_type Msg_text > +test.filt analyze status Engine-independent statistics collected > +test.filt analyze status OK > +test.acei analyze status Engine-independent statistics collected > +test.acei analyze status OK > +test.acli analyze status Engine-independent statistics collected > +test.acli analyze status OK > +set @save_optimizer_switch=@@optimizer_switch; > +set @save_join_cache_level=@@join_cache_level; > +set optimizer_switch='mrr=off'; > +set join_cache_level=2; > +set statement optimizer_switch='rowid_filter=off' for explain extended select t.id, fi.* > +from (acli t inner join acei a on a.aclid = t.id) > +inner join filt fi on a.id = fi.aceid > +where > +t.rid = 'B5FCC8C7111E4E3CBC21AAF5012F59C2' and > +t.tp = 121 and > +a.atp = 1 and > +fi.fh in (6311439873746261694,-397087483897438286, > +8518228073041491534,-5420422472375069774); > +id select_type table type possible_keys key key_len ref rows filtered Extra > +1 SIMPLE t index_merge PRIMARY,acli_rid,acli_tp acli_tp,acli_rid 2,767 NULL 2 100.00 Using intersect(acli_tp,acli_rid); Using where; Using index > +1 SIMPLE a ref PRIMARY,acei_aclid acei_aclid 8 test.t.id 1 100.00 Using where > +1 SIMPLE fi ref filt_aceid,filt_fh filt_aceid 8 test.a.id 1 17.14 Using where > +Warnings: > +Note 1003 select `test`.`t`.`id` AS `id`,`test`.`fi`.`id` AS `id`,`test`.`fi`.`aceid` AS `aceid`,`test`.`fi`.`clid` AS `clid`,`test`.`fi`.`fh` AS `fh` from `test`.`acli` `t` join `test`.`acei` `a` join `test`.`filt` `fi` where `test`.`t`.`tp` = 121 and `test`.`a`.`atp` = 1 and `test`.`fi`.`aceid` = `test`.`a`.`id` and `test`.`a`.`aclid` = `test`.`t`.`id` and `test`.`t`.`rid` = 'B5FCC8C7111E4E3CBC21AAF5012F59C2' and `test`.`fi`.`fh` in (6311439873746261694,-397087483897438286,8518228073041491534,-5420422472375069774) > +set statement optimizer_switch='rowid_filter=off' for select t.id, fi.* > +from (acli t inner join acei a on a.aclid = t.id) > +inner join filt fi on a.id = fi.aceid > +where > +t.rid = 'B5FCC8C7111E4E3CBC21AAF5012F59C2' and > +t.tp = 121 and > +a.atp = 1 and > +fi.fh in (6311439873746261694,-397087483897438286, > +8518228073041491534,-5420422472375069774); > +id id aceid clid fh > +3080602882609775594 3080602882609775600 3080602882609775598 1 6311439873746261694 > +3080602882609775594 3080602882609775601 3080602882609775598 1 6311439873746261694 > +set statement optimizer_switch='rowid_filter=on' for explain extended select t.id, fi.* > +from (acli t inner join acei a on a.aclid = t.id) > +inner join filt fi on a.id = fi.aceid > +where > +t.rid = 'B5FCC8C7111E4E3CBC21AAF5012F59C2' and > +t.tp = 121 and > +a.atp = 1 and > +fi.fh in (6311439873746261694,-397087483897438286, > +8518228073041491534,-5420422472375069774); > +id select_type table type possible_keys key key_len ref rows filtered Extra > +1 SIMPLE t index_merge PRIMARY,acli_rid,acli_tp acli_tp,acli_rid 2,767 NULL 2 100.00 Using intersect(acli_tp,acli_rid); Using where; Using index > +1 SIMPLE a ref PRIMARY,acei_aclid acei_aclid 8 test.t.id 1 100.00 Using where > +1 SIMPLE fi ref|filter filt_aceid,filt_fh filt_aceid|filt_fh 8|8 test.a.id 1 (17%) 17.14 Using where; Using rowid filter > +Warnings: > +Note 1003 select `test`.`t`.`id` AS `id`,`test`.`fi`.`id` AS `id`,`test`.`fi`.`aceid` AS `aceid`,`test`.`fi`.`clid` AS `clid`,`test`.`fi`.`fh` AS `fh` from `test`.`acli` `t` join `test`.`acei` `a` join `test`.`filt` `fi` where `test`.`t`.`tp` = 121 and `test`.`a`.`atp` = 1 and `test`.`fi`.`aceid` = `test`.`a`.`id` and `test`.`a`.`aclid` = `test`.`t`.`id` and `test`.`t`.`rid` = 'B5FCC8C7111E4E3CBC21AAF5012F59C2' and `test`.`fi`.`fh` in (6311439873746261694,-397087483897438286,8518228073041491534,-5420422472375069774) > +set statement optimizer_switch='rowid_filter=on' for select t.id, fi.* > +from (acli t inner join acei a on a.aclid = t.id) > +inner join filt fi on a.id = fi.aceid > +where > +t.rid = 'B5FCC8C7111E4E3CBC21AAF5012F59C2' and > +t.tp = 121 and > +a.atp = 1 and > +fi.fh in (6311439873746261694,-397087483897438286, > +8518228073041491534,-5420422472375069774); > +id id aceid clid fh > +3080602882609775594 3080602882609775600 3080602882609775598 1 6311439873746261694 > +3080602882609775594 3080602882609775601 3080602882609775598 1 6311439873746261694 > +set optimizer_switch='mrr=on'; > +set join_cache_level=6; > +set statement optimizer_switch='rowid_filter=off' for explain extended select t.id, fi.* > +from (acli t inner join acei a on a.aclid = t.id) > +inner join filt fi on a.id = fi.aceid > +where > +t.rid = 'B5FCC8C7111E4E3CBC21AAF5012F59C2' and > +t.tp = 121 and > +a.atp = 1 and > +fi.fh in (6311439873746261694,-397087483897438286, > +8518228073041491534,-5420422472375069774); > +id select_type table type possible_keys key key_len ref rows filtered Extra > +1 SIMPLE t index_merge PRIMARY,acli_rid,acli_tp acli_tp,acli_rid 2,767 NULL 2 100.00 Using intersect(acli_tp,acli_rid); Using where; Using index > +1 SIMPLE a ref PRIMARY,acei_aclid acei_aclid 8 test.t.id 1 100.00 Using where; Using join buffer (flat, BKA join); Rowid-ordered scan > +1 SIMPLE fi ref filt_aceid,filt_fh filt_aceid 8 test.a.id 1 17.14 Using where; Using join buffer (incremental, BKA join); Rowid-ordered scan > +Warnings: > +Note 1003 select `test`.`t`.`id` AS `id`,`test`.`fi`.`id` AS `id`,`test`.`fi`.`aceid` AS `aceid`,`test`.`fi`.`clid` AS `clid`,`test`.`fi`.`fh` AS `fh` from `test`.`acli` `t` join `test`.`acei` `a` join `test`.`filt` `fi` where `test`.`t`.`tp` = 121 and `test`.`a`.`atp` = 1 and `test`.`fi`.`aceid` = `test`.`a`.`id` and `test`.`a`.`aclid` = `test`.`t`.`id` and `test`.`t`.`rid` = 'B5FCC8C7111E4E3CBC21AAF5012F59C2' and `test`.`fi`.`fh` in (6311439873746261694,-397087483897438286,8518228073041491534,-5420422472375069774) > +set statement optimizer_switch='rowid_filter=off' for select t.id, fi.* > +from (acli t inner join acei a on a.aclid = t.id) > +inner join filt fi on a.id = fi.aceid > +where > +t.rid = 'B5FCC8C7111E4E3CBC21AAF5012F59C2' and > +t.tp = 121 and > +a.atp = 1 and > +fi.fh in (6311439873746261694,-397087483897438286, > +8518228073041491534,-5420422472375069774); > +id id aceid clid fh > +3080602882609775594 3080602882609775600 3080602882609775598 1 6311439873746261694 > +3080602882609775594 3080602882609775601 3080602882609775598 1 6311439873746261694 > +set statement optimizer_switch='rowid_filter=on' for explain extended select t.id, fi.* > +from (acli t inner join acei a on a.aclid = t.id) > +inner join filt fi on a.id = fi.aceid > +where > +t.rid = 'B5FCC8C7111E4E3CBC21AAF5012F59C2' and > +t.tp = 121 and > +a.atp = 1 and > +fi.fh in (6311439873746261694,-397087483897438286, > +8518228073041491534,-5420422472375069774); > +id select_type table type possible_keys key key_len ref rows filtered Extra > +1 SIMPLE t index_merge PRIMARY,acli_rid,acli_tp acli_tp,acli_rid 2,767 NULL 2 100.00 Using intersect(acli_tp,acli_rid); Using where; Using index > +1 SIMPLE a ref PRIMARY,acei_aclid acei_aclid 8 test.t.id 1 100.00 Using where; Using join buffer (flat, BKA join); Rowid-ordered scan > +1 SIMPLE fi ref|filter filt_aceid,filt_fh filt_aceid|filt_fh 8|8 test.a.id 1 (17%) 17.14 Using where; Using join buffer (incremental, BKA join); Rowid-ordered scan; Using rowid filter > +Warnings: > +Note 1003 select `test`.`t`.`id` AS `id`,`test`.`fi`.`id` AS `id`,`test`.`fi`.`aceid` AS `aceid`,`test`.`fi`.`clid` AS `clid`,`test`.`fi`.`fh` AS `fh` from `test`.`acli` `t` join `test`.`acei` `a` join `test`.`filt` `fi` where `test`.`t`.`tp` = 121 and `test`.`a`.`atp` = 1 and `test`.`fi`.`aceid` = `test`.`a`.`id` and `test`.`a`.`aclid` = `test`.`t`.`id` and `test`.`t`.`rid` = 'B5FCC8C7111E4E3CBC21AAF5012F59C2' and `test`.`fi`.`fh` in (6311439873746261694,-397087483897438286,8518228073041491534,-5420422472375069774) > +set statement optimizer_switch='rowid_filter=on' for select t.id, fi.* > +from (acli t inner join acei a on a.aclid = t.id) > +inner join filt fi on a.id = fi.aceid > +where > +t.rid = 'B5FCC8C7111E4E3CBC21AAF5012F59C2' and > +t.tp = 121 and > +a.atp = 1 and > +fi.fh in (6311439873746261694,-397087483897438286, > +8518228073041491534,-5420422472375069774); > +id id aceid clid fh > +3080602882609775594 3080602882609775600 3080602882609775598 1 6311439873746261694 > +3080602882609775594 3080602882609775601 3080602882609775598 1 6311439873746261694 > +set statement optimizer_switch='rowid_filter=on' for analyze format=json select t.id, fi.* > +from (acli t inner join acei a on a.aclid = t.id) > +inner join filt fi on a.id = fi.aceid > +where > +t.rid = 'B5FCC8C7111E4E3CBC21AAF5012F59C2' and > +t.tp = 121 and > +a.atp = 1 and > +fi.fh in (6311439873746261694,-397087483897438286, > +8518228073041491534,-5420422472375069774); > +ANALYZE > +{ > + "query_block": { > + "select_id": 1, > + "r_loops": 1, > + "r_total_time_ms": "REPLACED", > + "table": { > + "table_name": "t", > + "access_type": "index_merge", > + "possible_keys": ["PRIMARY", "acli_rid", "acli_tp"], > + "key_length": "2,767", > + "index_merge": { > + "intersect": { > + "range": { > + "key": "acli_tp", > + "used_key_parts": ["tp"] > + }, > + "range": { > + "key": "acli_rid", > + "used_key_parts": ["rid"] > + } > + } > + }, > + "r_loops": 1, > + "rows": 2, > + "r_rows": 3, > + "r_total_time_ms": "REPLACED", > + "filtered": 100, > + "r_filtered": 100, > + "attached_condition": "t.tp = 121 and t.rid = 'B5FCC8C7111E4E3CBC21AAF5012F59C2'", > + "using_index": true > + }, > + "block-nl-join": { > + "table": { > + "table_name": "a", > + "access_type": "ref", > + "possible_keys": ["PRIMARY", "acei_aclid"], > + "key": "acei_aclid", > + "key_length": "8", > + "used_key_parts": ["aclid"], > + "ref": ["test.t.id"], > + "r_loops": 1, > + "rows": 1, > + "r_rows": 3, > + "r_total_time_ms": "REPLACED", > + "filtered": 100, > + "r_filtered": 100 > + }, > + "buffer_type": "flat", > + "buffer_size": "8Kb", > + "join_type": "BKA", > + "mrr_type": "Rowid-ordered scan", > + "attached_condition": "a.atp = 1", > + "r_filtered": 100 > + }, > + "block-nl-join": { > + "table": { > + "table_name": "fi", > + "access_type": "ref", > + "possible_keys": ["filt_aceid", "filt_fh"], > + "key": "filt_aceid", > + "key_length": "8", > + "used_key_parts": ["aceid"], > + "ref": ["test.a.id"], > + "rowid_filter": { > + "range": { > + "key": "filt_fh", > + "used_key_parts": ["fh"] > + }, > + "rows": 6, > + "selectivity_pct": 17.143, > + "r_rows": 5, > + "r_selectivity_pct": 40, > + "r_buffer_size": "REPLACED", > + "r_filling_time_ms": "REPLACED" > + }, > + "r_loops": 1, > + "rows": 1, > + "r_rows": 2, > + "r_total_time_ms": "REPLACED", > + "filtered": 17.143, > + "r_filtered": 100 > + }, > + "buffer_type": "incremental", > + "buffer_size": "603", > + "join_type": "BKA", > + "mrr_type": "Rowid-ordered scan", > + "attached_condition": "fi.fh in (6311439873746261694,-397087483897438286,8518228073041491534,-5420422472375069774)", > + "r_filtered": 100 > + } > + } > +} > +set optimizer_switch=@save_optimizer_switch; > +set join_cache_level=@save_join_cache_level; > +drop table filt, acei, acli; > +set global innodb_stats_persistent= @stats.save; > diff --git a/mysql-test/main/rowid_filter_innodb.test b/mysql-test/main/rowid_filter_innodb.test > index 30e0ede..74349b8 100644 > --- a/mysql-test/main/rowid_filter_innodb.test > +++ b/mysql-test/main/rowid_filter_innodb.test > @@ -381,3 +381,156 @@ ORDER BY pk LIMIT 1; > > DROP TABLE t1; > SET global innodb_stats_persistent= @stats.save; > + > +--echo # > +--echo # MDEV-21610: Using rowid filter with BKA+MRR > +--echo # > + > +set @stats.save= @@innodb_stats_persistent; > +set global innodb_stats_persistent=on; > + > +CREATE TABLE acli ( > + id bigint(20) NOT NULL, > + rid varchar(255) NOT NULL, > + tp smallint(6) NOT NULL DEFAULT 0, > + PRIMARY KEY (id), > + KEY acli_rid (rid), > + KEY acli_tp (tp) > +) ENGINE=InnoDB DEFAULT CHARSET=utf8; > + > +insert into acli(id,rid,tp) values > +(184929059698905997,'ABABABABABABABABAB',103), > +(184929059698905998,'ABABABABABABABABAB',121), > +(283586039035985921,'00000000000000000000000000000000',103), > +(2216474704108064678,'020BED6D07B741CE9B10AB2200FEF1DF',103), > +(2216474704108064679,'020BED6D07B741CE9B10AB2200FEF1DF',121), > +(3080602882609775593,'B5FCC8C7111E4E3CBC21AAF5012F59C2',103), > +(3080602882609775594,'B5FCC8C7111E4E3CBC21AAF5012F59C2',121), > +(3080602882609776594,'B5FCC8C7111E4E3CBC21AAF5012F59C2',121), > +(3080602882609777595,'B5FCC8C7111E4E3CBC21AAF5012F59C2',121), > +(4269412446747236214,'SCSCSCSCSCSCSCSC',103), > +(4269412446747236215,'SCSCSCSCSCSCSCSC',121), > +(6341490487802728356,'6072D47E513F4A4794BBAB2200FDB67D',103), > +(6341490487802728357,'6072D47E513F4A4794BBAB2200FDB67D',121); > + > +CREATE TABLE acei ( > + id bigint(20) NOT NULL, > + aclid bigint(20) NOT NULL DEFAULT 0, > + atp smallint(6) NOT NULL DEFAULT 0, > + clus smallint(6) NOT NULL DEFAULT 0, > + PRIMARY KEY (id), > + KEY acei_aclid (aclid), > + KEY acei_clus (clus) > +) ENGINE=InnoDB DEFAULT CHARSET=utf8; > + > +insert into acei(id,aclid,atp,clus) values > +(184929059698905999,184929059698905997,0,1), > +(184929059698906000,184929059698905997,0,1), > +(184929059698906001,184929059698905997,1,1), > +(184929059698906002,184929059698905998,1,1), > +(283586039035985922,283586039035985921,1,1), > +(2216474704108064684,2216474704108064678,0,1), > +(2216474704108064685,2216474704108064678,0,1), > +(2216474704108064686,2216474704108064678,1,1), > +(2216474704108064687,2216474704108064679,1,1), > +(3080602882609775595,3080602882609775593,0,1), > +(3080602882609775596,3080602882609775593,0,1), > +(3080602882609775597,3080602882609775593,1,1), > +(3080602882609775598,3080602882609775594,1,1), > +(3080602882609776595,3080602882609776594,1,1), > +(3080602882609777596,3080602882609777595,1,1), > +(4269412446747236216,4269412446747236214,0,1), > +(4269412446747236217,4269412446747236214,0,1), > +(4269412446747236218,4269412446747236214,1,1), > +(4269412446747236219,4269412446747236215,1,1), > +(6341490487802728358,6341490487802728356,0,1), > +(6341490487802728359,6341490487802728356,0,1), > +(6341490487802728360,6341490487802728356,1,1), > +(6341490487802728361,6341490487802728357,1,1); > + > +CREATE TABLE filt ( > + id bigint(20) NOT NULL, > + aceid bigint(20) NOT NULL DEFAULT 0, > + clid smallint(6) NOT NULL DEFAULT 0, > + fh bigint(20) NOT NULL DEFAULT 0, > + PRIMARY KEY (id), > + KEY filt_aceid (aceid), > + KEY filt_clid (clid), > + KEY filt_fh (fh) > +) ENGINE=InnoDB DEFAULT CHARSET=utf8; > + > +insert into filt(id,aceid,clid,fh) values > +(184929059698905999,184929059698905999,1,8948400944397203540), > +(184929059698906000,184929059698906000,1,-3516039679025944536), > +(184929059698906001,184929059698906001,1,-3516039679025944536), > +(184929059698906002,184929059698906001,1,2965370193075218252), > +(184929059698906003,184929059698906001,1,8948400944397203540), > +(184929059698906004,184929059698906002,1,2478709353550777738), > +(283586039035985922,283586039035985922,1,5902600816362013271), > +(2216474704108064686,2216474704108064684,1,8948400944397203540), > +(2216474704108064687,2216474704108064685,1,-7244708939311117030), > +(2216474704108064688,2216474704108064686,1,-7244708939311117030), > +(2216474704108064689,2216474704108064686,1,7489060986210282479), > +(2216474704108064690,2216474704108064686,1,8948400944397203540), > +(2216474704108064691,2216474704108064687,1,-3575268945274980038), > +(3080602882609775595,3080602882609775595,1,8948400944397203540), > +(3080602882609775596,3080602882609775596,1,-5420422472375069774), > +(3080602882609775597,3080602882609775597,1,-5420422472375069774), > +(3080602882609775598,3080602882609775597,1,8518228073041491534), > +(3080602882609775599,3080602882609775597,1,8948400944397203540), > +(3080602882609775600,3080602882609775598,1,6311439873746261694), > +(3080602882609775601,3080602882609775598,1,6311439873746261694), > +(3080602882609776595,3080602882609776595,1,-661101805245999843), > +(3080602882609777596,3080602882609777596,1,-661101805245999843), > +(3080602882609777597,3080602882609777596,1,2216865386202464067), > +(4269412446747236216,4269412446747236216,1,8948400944397203540), > +(4269412446747236217,4269412446747236217,1,-1143096194892676000), > +(4269412446747236218,4269412446747236218,1,-1143096194892676000), > +(4269412446747236219,4269412446747236218,1,5313391811364818290), > +(4269412446747236220,4269412446747236218,1,8948400944397203540), > +(4269412446747236221,4269412446747236219,1,7624499822621753835), > +(6341490487802728358,6341490487802728358,1,8948400944397203540), > +(6341490487802728359,6341490487802728359,1,8141092449587136068), > +(6341490487802728360,6341490487802728360,1,8141092449587136068), > +(6341490487802728361,6341490487802728360,1,1291319099896431785), > +(6341490487802728362,6341490487802728360,1,8948400944397203540), > +(6341490487802728363,6341490487802728361,1,6701841652906431497); > + > +analyze table filt, acei, acli; > + > +let $q= > +select t.id, fi.* > +from (acli t inner join acei a on a.aclid = t.id) > + inner join filt fi on a.id = fi.aceid > + where > + t.rid = 'B5FCC8C7111E4E3CBC21AAF5012F59C2' and > + t.tp = 121 and > + a.atp = 1 and > + fi.fh in (6311439873746261694,-397087483897438286, > + 8518228073041491534,-5420422472375069774); > + > +set @save_optimizer_switch=@@optimizer_switch; > +set @save_join_cache_level=@@join_cache_level; > + > +set optimizer_switch='mrr=off'; > +set join_cache_level=2; > +eval $without_filter explain extended $q; > +eval $without_filter $q; > +eval $with_filter explain extended $q; > +eval $with_filter $q; > + > +set optimizer_switch='mrr=on'; > +set join_cache_level=6; > +eval $without_filter explain extended $q; > +eval $without_filter $q; > +eval $with_filter explain extended $q; > +eval $with_filter $q; > +--source include/analyze-format.inc > +eval $with_filter analyze format=json $q; > + > +set optimizer_switch=@save_optimizer_switch; > +set join_cache_level=@save_join_cache_level; > + > +drop table filt, acei, acli; > + > +set global innodb_stats_persistent= @stats.save; > diff --git a/sql/multi_range_read.cc b/sql/multi_range_read.cc > index 7e4c2ed..daeb53d 100644 > --- a/sql/multi_range_read.cc > +++ b/sql/multi_range_read.cc > @@ -702,7 +702,8 @@ static int rowid_cmp_reverse(void *file, uchar *a, uchar *b) > int Mrr_ordered_rndpos_reader::init(handler *h_arg, > Mrr_index_reader *index_reader_arg, > uint mode, > - Lifo_buffer *buf) > + Lifo_buffer *buf, > + Rowid_filter *filter) > { > file= h_arg; > index_reader= index_reader_arg; > @@ -710,19 +711,7 @@ int Mrr_ordered_rndpos_reader::init(handler *h_arg, > is_mrr_assoc= !MY_TEST(mode & HA_MRR_NO_ASSOCIATION); > index_reader_exhausted= FALSE; > index_reader_needs_refill= TRUE; > - > - /* > - Currently usage of a rowid filter within InnoDB engine is not supported > - if the table is accessed by the primary key. > - With optimizer switches ''mrr' and 'mrr_sort_keys' are both enabled > - any access by a secondary index is converted to the rndpos access. In > - InnoDB the rndpos access is always uses the primary key. > - Do not use pushed rowid filter if the table is accessed actually by the > - primary key. Use the rowid filter outside the engine code (see > - Mrr_ordered_rndpos_reader::refill_from_index_reader). > - */ > - if (file->pushed_rowid_filter && file->primary_key_is_clustered()) > - file->cancel_pushed_rowid_filter(); > + rowid_filter= filter; > > return 0; > } > @@ -817,10 +806,8 @@ int Mrr_ordered_rndpos_reader::refill_from_index_reader() > index_reader->position(); > > /* > - If the built rowid filter cannot be used at the engine level use it here. > + If the built rowid filter cannot be used at the engine level, use it here. > */ > - Rowid_filter *rowid_filter= > - file->get_table()->reginfo.join_tab->rowid_filter; > if (rowid_filter && !file->pushed_rowid_filter && > !rowid_filter->check((char *)index_rowid)) > continue; > @@ -967,6 +954,7 @@ int DsMrr_impl::dsmrr_init(handler *h_arg, RANGE_SEQ_IF *seq_funcs, > handler *h_idx; > Mrr_ordered_rndpos_reader *disk_strategy= NULL; > bool do_sort_keys= FALSE; > + Rowid_filter *rowid_filter= NULL; > DBUG_ENTER("DsMrr_impl::dsmrr_init"); > /* > index_merge may invoke a scan on an object for which dsmrr_info[_const] > @@ -1015,6 +1003,21 @@ int DsMrr_impl::dsmrr_init(handler *h_arg, RANGE_SEQ_IF *seq_funcs, > if (!(keyno == table->s->primary_key && h_idx->primary_key_is_clustered())) > { > strategy= disk_strategy= &reader_factory.ordered_rndpos_reader; > + if (h_arg->pushed_rowid_filter) > + { > + /* > + Currently usage of a rowid filter within InnoDB engine is not supported > + if the table is accessed by the primary key. > + With optimizer switches ''mrr' and 'mrr_sort_keys' are both enabled > + any access by a secondary index is converted to the rndpos access. In > + InnoDB the rndpos access is always uses the primary key. > + Do not use pushed rowid filter if the table is accessed actually by the > + primary key. Use the rowid filter outside the engine code (see > + Mrr_ordered_rndpos_reader::refill_from_index_reader). > + */ > + rowid_filter= h_arg->pushed_rowid_filter; > + h_arg->cancel_pushed_rowid_filter(); > + } > } > > full_buf= buf->buffer; > @@ -1101,7 +1104,7 @@ int DsMrr_impl::dsmrr_init(handler *h_arg, RANGE_SEQ_IF *seq_funcs, > n_ranges, mode, &keypar, key_buffer, > &buf_manager)) || > (res= disk_strategy->init(primary_file, index_strategy, mode, > - &rowid_buffer))) > + &rowid_buffer, rowid_filter))) > { > goto error; > } > diff --git a/sql/multi_range_read.h b/sql/multi_range_read.h > index 0473fef..6be9537 100644 > --- a/sql/multi_range_read.h > +++ b/sql/multi_range_read.h > @@ -364,7 +364,7 @@ class Mrr_ordered_rndpos_reader : public Mrr_reader > { > public: > int init(handler *file, Mrr_index_reader *index_reader, uint mode, > - Lifo_buffer *buf); > + Lifo_buffer *buf, Rowid_filter *filter); > int get_next(range_id_t *range_info); > int refill_buffer(bool initial); > private: > @@ -399,6 +399,9 @@ class Mrr_ordered_rndpos_reader : public Mrr_reader > /* Buffer to store (rowid, range_id) pairs */ > Lifo_buffer *rowid_buffer; > > + /* Rowid filter to be checked against (if any) */ > + Rowid_filter *rowid_filter; > + > int refill_from_index_reader(); > }; > > diff --git a/sql/opt_range.cc b/sql/opt_range.cc > index c47da28..5f034c6 100644 > --- a/sql/opt_range.cc > +++ b/sql/opt_range.cc > @@ -2902,7 +2902,8 @@ int SQL_SELECT::test_quick_select(THD *thd, key_map keys_to_use, > remove_nonrange_trees(&param, tree); > > /* Get best 'range' plan and prepare data for making other plans */ > - if ((range_trp= get_key_scans_params(&param, tree, FALSE, TRUE, > + if ((range_trp= get_key_scans_params(&param, tree, > + only_single_index_range_scan, TRUE, > best_read_time))) > { > best_trp= range_trp; > diff --git a/sql/sql_join_cache.cc b/sql/sql_join_cache.cc > index 3a509b3..e9ad538 100644 > --- a/sql/sql_join_cache.cc > +++ b/sql/sql_join_cache.cc > @@ -2248,6 +2248,8 @@ enum_nested_loop_state JOIN_CACHE::join_matching_records(bool skip_last) > if ((rc= join_tab_execution_startup(join_tab)) < 0) > goto finish2; > > + join_tab->build_range_rowid_filter_if_needed(); > + > /* Prepare to retrieve all records of the joined table */ > if (unlikely((error= join_tab_scan->open()))) > { > _______________________________________________ > commits mailing list > commits(a)mariadb.org > https://lists.askmonty.org/cgi-bin/mailman/listinfo/commits -- BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

[Maria-developers] MDEV-7381
by Gabriel Ciciliani 11 Feb '20

11 Feb '20

Hi MariaDB developers, I want to start working on this ticket but I can't assing it to me with my current JIRA privileges. Someone higher in the chain should do it for me or I nee to get my JIRA account fixed? Thanks!

2 1

Re: [Maria-developers] 6887f16b61a: MDEV-17399 Add support for JSON_TABLE.
by Sergei Golubchik 06 Feb '20

06 Feb '20

Hi, Alexey! There isn't much to review yet, I don't see how different parts will work together yet. But here you are, some preliminary comments. On Feb 06, Alexey Botchkov wrote: > revision-id: 6887f16b61a (mariadb-10.5.0-153-g6887f16b61a) > parent(s): 74f76206369 > author: Alexey Botchkov <holyfoot(a)mariadb.com> > committer: Alexey Botchkov <holyfoot(a)mariadb.com> > timestamp: 2020-02-04 17:12:55 +0400 > message: > > MDEV-17399 Add support for JSON_TABLE. > > Syntax for the JSON_TABLE added. Standard JSON_TABLE syntax is quite complex. You are doing a subset of it, I presume. It would be good to specify here, in the comment, what syntax exactly are you implementing. > --- > sql/lex.h | 5 ++ > sql/sql_digest.cc | 1 + > sql/sql_lex.h | 3 + > sql/sql_yacc.yy | 164 ++++++++++++++++++++++++++++++++++++++++++++++++++- > sql/table.cc | 1 + > sql/table.h | 2 + > sql/table_function.h | 89 ++++++++++++++++++++++++++++ > 7 files changed, 264 insertions(+), 1 deletion(-) > > diff --git a/sql/lex.h b/sql/lex.h > index f3fc1513369..774f9c1517d 100644 > --- a/sql/lex.h > +++ b/sql/lex.h > @@ -208,6 +208,7 @@ static SYMBOL symbols[] = { > { "ELSE", SYM(ELSE)}, > { "ELSEIF", SYM(ELSEIF_MARIADB_SYM)}, > { "ELSIF", SYM(ELSIF_MARIADB_SYM)}, > + { "EMPTY", SYM(EMPTY_SYM)}, > { "ENABLE", SYM(ENABLE_SYM)}, > { "ENCLOSED", SYM(ENCLOSED)}, > { "END", SYM(END)}, > @@ -414,6 +415,7 @@ static SYMBOL symbols[] = { > { "NATIONAL", SYM(NATIONAL_SYM)}, > { "NATURAL", SYM(NATURAL)}, > { "NCHAR", SYM(NCHAR_SYM)}, > + { "NESTED", SYM(NESTED_SYM)}, > { "NEVER", SYM(NEVER_SYM)}, > { "NEW", SYM(NEW_SYM)}, > { "NEXT", SYM(NEXT_SYM)}, > @@ -448,6 +450,7 @@ static SYMBOL symbols[] = { > { "OPTIONALLY", SYM(OPTIONALLY)}, > { "OR", SYM(OR_SYM)}, > { "ORDER", SYM(ORDER_SYM)}, > + { "ORDINALITY", SYM(ORDINALITY_SYM)}, > { "OTHERS", SYM(OTHERS_MARIADB_SYM)}, > { "OUT", SYM(OUT_SYM)}, > { "OUTER", SYM(OUTER)}, > @@ -460,6 +463,7 @@ static SYMBOL symbols[] = { > { "PAGE_CHECKSUM", SYM(PAGE_CHECKSUM_SYM)}, > { "PARSER", SYM(PARSER_SYM)}, > { "PARSE_VCOL_EXPR", SYM(PARSE_VCOL_EXPR_SYM)}, > + { "PATH", SYM(PATH_SYM)}, > { "PERIOD", SYM(PERIOD_SYM)}, > { "PARTIAL", SYM(PARTIAL)}, > { "PARTITION", SYM(PARTITION_SYM)}, > @@ -742,6 +746,7 @@ static SYMBOL sql_functions[] = { > { "FIRST_VALUE", SYM(FIRST_VALUE_SYM)}, > { "GROUP_CONCAT", SYM(GROUP_CONCAT_SYM)}, > { "JSON_ARRAYAGG", SYM(JSON_ARRAYAGG_SYM)}, > + { "JSON_TABLE", SYM(JSON_TABLE_SYM)}, > { "JSON_OBJECTAGG", SYM(JSON_OBJECTAGG_SYM)}, > { "LAG", SYM(LAG_SYM)}, > { "LEAD", SYM(LEAD_SYM)}, you've added them all as reserved words, is it intentonal? > diff --git a/sql/sql_lex.h b/sql/sql_lex.h > index 308bebd9fa1..591de1e1583 100644 > --- a/sql/sql_lex.h > +++ b/sql/sql_lex.h > @@ -33,6 +33,7 @@ > #include "sql_tvc.h" > #include "item.h" > #include "sql_limit.h" // Select_limit_counters > +#include "table_function.h" // Json_table_column > > /* Used for flags of nesting constructs */ > #define SELECT_NESTING_MAP_SIZE 64 > @@ -3249,6 +3250,8 @@ struct LEX: public Query_tables_list > SQL_I_List<ORDER> proc_list; > SQL_I_List<TABLE_LIST> auxiliary_table_list, save_list; > Column_definition *last_field; > + Json_table_column *cur_json_table_column, *json_table_column_nest; > + Table_function_json_table *json_table; Did you have to put everything in LEX? It's pretty big already and it's never fully used, its members allocate memory all together at the same time, but used for different parts of the grammar, different statements. Could you try to pass that through the parser stack? > Item_sum *in_sum_func; > udf_func udf; > HA_CHECK_OPT check_opt; // check/repair options > diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy > index c00ba42846a..5ef1bbd0721 100644 > --- a/sql/sql_yacc.yy > +++ b/sql/sql_yacc.yy > @@ -207,6 +208,8 @@ void _CONCAT_UNDERSCORED(turn_parser_debug_on,yyparse)() > Lex_for_loop_st for_loop; > Lex_for_loop_bounds_st for_loop_bounds; > Lex_trim_st trim; > + Json_table_column::On_response json_on_response; > + > vers_history_point_t vers_history_point; > struct > { > @@ -225,6 +228,7 @@ void _CONCAT_UNDERSCORED(turn_parser_debug_on,yyparse)() > > /* pointers */ > Create_field *create_field; > + Json_table_column *json_table_column; > Spvar_definition *spvar_definition; > Row_definition_list *spvar_definition_list; > const Type_handler *type_handler; > @@ -308,6 +312,7 @@ void _CONCAT_UNDERSCORED(turn_parser_debug_on,yyparse)() > enum vers_kind_t vers_range_unit; > enum Column_definition::enum_column_versioning vers_column_versioning; > enum plsql_cursor_attr_t plsql_cursor_attr; > + enum Json_table_column::enum_on_type json_on_type; > } > > %{ > @@ -489,6 +494,7 @@ End SQL_MODE_ORACLE_SPECIFIC */ > %token <kwd> ELSEIF_MARIADB_SYM > %token <kwd> ELSE /* SQL-2003-R */ > %token <kwd> ELSIF_ORACLE_SYM /* PLSQL-R */ > +%token <kwd> EMPTY_SYM /* SQL-2016-R */ > %token <kwd> ENCLOSED > %token <kwd> ESCAPED > %token <kwd> EXCEPT_SYM /* SQL-2003-R */ > @@ -506,6 +512,7 @@ End SQL_MODE_ORACLE_SPECIFIC */ > %token <kwd> GRANT /* SQL-2003-R */ > %token <kwd> GROUP_CONCAT_SYM > %token <rwd> JSON_ARRAYAGG_SYM > +%token <rwd> JSON_TABLE_SYM > %token <rwd> JSON_OBJECTAGG_SYM > %token <kwd> GROUP_SYM /* SQL-2003-R */ > %token <kwd> HAVING /* SQL-2003-R */ > @@ -564,6 +571,7 @@ End SQL_MODE_ORACLE_SPECIFIC */ > %token <kwd> MOD_SYM /* SQL-2003-N */ > %token <kwd> NATURAL /* SQL-2003-R */ > %token <kwd> NEG > +%token <kwd> NESTED_SYM /* SQL-2003-N */ No, NESTED is not in the SQL:2003 list of non-reserved words. Please specify a correct comment tag for every keyword you're adding here. > %token <kwd> NOT_SYM /* SQL-2003-R */ > %token <kwd> NO_WRITE_TO_BINLOG > %token <kwd> NOW_SYM > @@ -575,6 +583,7 @@ End SQL_MODE_ORACLE_SPECIFIC */ > %token <kwd> OPTIMIZE > %token <kwd> OPTIONALLY > %token <kwd> ORDER_SYM /* SQL-2003-R */ > +%token <kwd> ORDINALITY_SYM /* SQL-2003-N */ > %token <kwd> OR_SYM /* SQL-2003-R */ > %token <kwd> OTHERS_ORACLE_SYM /* SQL-2011-N, PLSQL-R */ > %token <kwd> OUTER > @@ -585,6 +594,7 @@ End SQL_MODE_ORACLE_SPECIFIC */ > %token <kwd> PAGE_CHECKSUM_SYM > %token <kwd> PARSE_VCOL_EXPR_SYM > %token <kwd> PARTITION_SYM /* SQL-2003-R */ > +%token <kwd> PATH_SYM /* SQL-2003-N */ > %token <kwd> PERCENTILE_CONT_SYM > %token <kwd> PERCENTILE_DISC_SYM > %token <kwd> PERCENT_RANK_SYM > @@ -1336,12 +1346,16 @@ End SQL_MODE_ORACLE_SPECIFIC */ > > %type <type_handler> int_type real_type > > +%type <json_on_type> json_empty_or_error > +%type <json_on_response> json_on_response > + > %type <Lex_field_type> type_with_opt_collate field_type > field_type_numeric > field_type_string > field_type_lob > field_type_temporal > field_type_misc > + json_table_field_type > > %type <Lex_dyncol_type> opt_dyncol_type dyncol_type > numeric_dyncol_type temporal_dyncol_type string_dyncol_type > @@ -1487,7 +1501,7 @@ End SQL_MODE_ORACLE_SPECIFIC */ > table_primary_derived table_primary_derived_opt_parens > derived_table_list table_reference_list_parens > nested_table_reference_list join_table_parens > - update_table_list > + update_table_list table_function > %type <date_time_type> date_time_type; > %type <interval> interval > > @@ -1642,6 +1656,9 @@ End SQL_MODE_ORACLE_SPECIFIC */ > opt_delete_gtid_domain > asrow_attribute > opt_constraint_no_id > + json_table_columns_clause json_table_columns_list json_table_column > + json_table_column_type json_opt_on_empty_or_error > + > > %type <NONE> call sp_proc_stmts sp_proc_stmts1 sp_proc_stmt > %type <NONE> sp_if_then_statements sp_case_then_statements > @@ -11320,6 +11337,150 @@ join_table_list: > derived_table_list { MYSQL_YYABORT_UNLESS($$=$1); } > ; > > +json_table_columns_clause: > + COLUMNS '(' json_table_columns_list ')' > + {} > + ; > + > +json_table_columns_list: > + json_table_column > + | json_table_columns_list ',' json_table_column > + {} > + ; > + > +json_table_column: > + ident > + { > + LEX *lex=Lex; > + Create_field *f= new (thd->mem_root) Create_field(); > + > + if (unlikely(check_string_char_length(&$1, 0, NAME_CHAR_LEN, > + system_charset_info, 1))) > + my_yyabort_error((ER_TOO_LONG_IDENT, MYF(0), $1.str)); > + > + lex->cur_json_table_column= > + new (thd->mem_root) Json_table_column(f, lex->json_table_column_nest); > + > + if (unlikely(!f || !lex->cur_json_table_column)) > + MYSQL_YYABORT; > + > + lex->init_last_field(f, &$1, NULL); > + } > + json_table_column_type > + { > + LEX *lex=Lex; > + lex->json_table->m_columns.push_back(lex->cur_json_table_column, > + thd->mem_root); > + } > + ; > + > +json_table_column_type: > + FOR_SYM ORDINALITY_SYM > + { > + Lex->cur_json_table_column->set(Json_table_column::FOR_ORDINALITY); > + } > + | json_table_field_type PATH_SYM TEXT_STRING_sys > + json_opt_on_empty_or_error > + { > + Lex->last_field->set_attributes(thd, $1, Lex->charset, > + COLUMN_DEFINITION_TABLE_FIELD); > + Lex->cur_json_table_column->set(Json_table_column::PATH, &$3); > + } > + | json_table_field_type EXISTS PATH_SYM TEXT_STRING_sys > + { > + Lex->last_field->set_attributes(thd, $1, Lex->charset, > + COLUMN_DEFINITION_TABLE_FIELD); > + Lex->cur_json_table_column->set(Json_table_column::EXISTS_PATH, &$4); > + } > + | NESTED_SYM PATH_SYM TEXT_STRING_sys > + { > + LEX *lex=Lex; > + lex->cur_json_table_column->set(Json_table_column::NESTED_PATH); > + lex->json_table_column_nest= lex->cur_json_table_column; > + } > + json_table_columns_clause > + { > + LEX *lex=Lex; > + lex->cur_json_table_column= lex->json_table_column_nest; > + lex->json_table_column_nest= lex->cur_json_table_column->m_nest; > + } > + ; > + > +json_table_field_type: > + field_type_numeric > + | field_type_temporal > + | field_type_string > + ; > + > +json_opt_on_empty_or_error: > + /* none */ > + {} > + | json_on_response ON json_empty_or_error json_opt_on_empty_or_error > + { > + if ($3 == Json_table_column::ON_EMPTY) > + { > + Lex->cur_json_table_column->m_on_empty= $1; > + } > + else /* ON_ERROR */ > + { > + Lex->cur_json_table_column->m_on_error= $1; > + } > + } > + ; > + > +json_on_response: > + ERROR_SYM > + { > + $$.m_response= Json_table_column::RESPONSE_ERROR; > + } > + | NULL_SYM > + { > + $$.m_response= Json_table_column::RESPONSE_NULL; > + } > + | DEFAULT TEXT_STRING_sys > + { > + $$.m_response= Json_table_column::RESPONSE_DEFAULT; > + $$.m_default= $1; > + } > + ; > + > +json_empty_or_error: > + ERROR_SYM > + { > + $$= Json_table_column::ON_EMPTY; > + } > + | EMPTY_SYM > + { > + $$= Json_table_column::ON_ERROR; > + } > + ; > + > +table_function: > + JSON_TABLE_SYM '(' expr ',' TEXT_STRING_sys difficult to see what you're parsing without a syntax specs in the commit comment. > + { > + Table_function_json_table *jt= > + new (thd->mem_root) Table_function_json_table($3, &$5); > + if (unlikely(!jt)) > + MYSQL_YYABORT; > + Lex->json_table= jt; > + Lex->json_table_column_nest= NULL; > + } > + json_table_columns_clause ')' AS ident_table_alias > + { > +LEX_CSTRING tn; > + SELECT_LEX *sel= Select; > + sel->table_join_options= 0; > +tn.str= "t1"; > +tn.length= 2; what's that? just to make the code compile for now? why not to use $10 - this is your ident_table_alias, currently it's ignored. > + if (!($$= Select->add_table_to_list(thd, > + new (thd->mem_root) Table_ident(&tn), &tn, > + Select->get_table_join_options(), > + YYPS->m_lock_type, > + YYPS->m_mdl_type))) > + MYSQL_YYABORT; > + } > + ; > + > /* > The ODBC escape syntax for Outer Join is: '{' OJ join_table '}' > The parser does not define OJ as a token, any ident is accepted > @@ -11527,6 +11688,7 @@ table_factor: > $$= $1; > } > | table_reference_list_parens { $$= $1; } > + | table_function { $$= $1; } > ; > > table_primary_ident_opt_parens: Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

[Maria-developers] [DISCUSS] Don't see ARM release information in MariaDB download page
by bo zhaobo 03 Feb '20

03 Feb '20

Hi Team, I want to make clear a thing about the Mariadb download page. Hope team could give some advices or help. As I had sent out an discussion to maria-docs, but didn't get some help. So I think I may get something in dev channel. When I want to download a aarch64 mariadb package, I'm failed to search any information in the content of page [1]. But I tried to search the RPM/DEB package, such as in '*Red Hat, Fedora, and CentOS Packages*' and '*Debian and Ubuntu Packages*', and I can see there are ARM(aarch64/arm64) specific packages in these linked pages [2]. And documentation in [1] just say OS/CPU with "*RedHat/CentOS/Fedora (x86, x86_64, ppc64, ppc64le)*". It seems that there is difference between the documentation with real released packages. That's why I'm here to want to make it clear and discuss with team. *How about add 'aarch64' in the OS/CPU description?* I think that would be great if community could fix that, because it is good for users. They could get what they want from MariaDB download page(MariaDB docs related). Thanks very much. BR ZhaoBo [1] https://downloads.mariadb.org/mariadb/10.5.0/ [2] http://ftp.igh.cnrs.fr/pub/mariadb//mariadb-10.5.0/yum/ http://ftp.igh.cnrs.fr/pub/mariadb//mariadb-10.5.0/repo/ [image: Mailtrack] <https://mailtrack.io?utm_source=gmail&utm_medium=signature&utm_campaign=sig…> Sender notified by Mailtrack <https://mailtrack.io?utm_source=gmail&utm_medium=signature&utm_campaign=sig…> 20/01/06 下午07:31:54

3 7

778e96749bc: MDEV-20923:UBSAN: member access within address … which does not point to an object of type 'xid_count_per_binlog'
by sujatha 23 Jan '20

23 Jan '20

revision-id: 778e96749bcadc5528bb7f214a711272f7413b96 (mariadb-10.1.43-66-g778e96749bc) parent(s): 982294ac1680938ac9223fb64a64e21f0cbc322a author: Sujatha committer: Sujatha timestamp: 2020-01-23 16:17:55 +0530 message: MDEV-20923:UBSAN: member access within address … which does not point to an object of type 'xid_count_per_binlog' Problem: ------- Accessing a member within 'xid_count_per_binlog' structure results in following error when 'UBSAN' is enabled. member access within address 0xXXX which does not point to an object of type 'xid_count_per_binlog' Analysis: --------- The problem appears to be that no constructor for 'xid_count_per_binlog' is being called, and thus the vtable will not be initialized. Fix: --- Defined a parameterized constructor for 'xid_count_per_binlog' class. --- sql/log.cc | 27 +++++++++++---------------- sql/log.h | 13 ++++++++++++- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/sql/log.cc b/sql/log.cc index acf1f8f8a9c..0efef6d1e29 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -3216,7 +3216,7 @@ void MYSQL_BIN_LOG::cleanup() DBUG_ASSERT(!binlog_xid_count_list.head()); WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::cleanup(): Removing xid_list_entry " "for %s (%lu)", b); - my_free(b); + delete b; } mysql_mutex_destroy(&LOCK_log); @@ -3580,17 +3580,9 @@ bool MYSQL_BIN_LOG::open(const char *log_name, */ uint off= dirname_length(log_file_name); uint len= strlen(log_file_name) - off; - char *entry_mem, *name_mem; - if (!(new_xid_list_entry = (xid_count_per_binlog *) - my_multi_malloc(MYF(MY_WME), - &entry_mem, sizeof(xid_count_per_binlog), - &name_mem, len, - NULL))) + new_xid_list_entry= new xid_count_per_binlog(log_file_name+off, (int)len); + if (!new_xid_list_entry) goto err; - memcpy(name_mem, log_file_name+off, len); - new_xid_list_entry->binlog_name= name_mem; - new_xid_list_entry->binlog_name_len= len; - new_xid_list_entry->xid_count= 0; /* Find the name for the Initial binlog checkpoint. @@ -3607,7 +3599,10 @@ bool MYSQL_BIN_LOG::open(const char *log_name, mysql_mutex_unlock(&LOCK_xid_list); if (!b) b= new_xid_list_entry; - strmake(buf, b->binlog_name, b->binlog_name_len); + if (b->binlog_name) + strmake(buf, b->binlog_name, b->binlog_name_len); + else + goto err; Binlog_checkpoint_log_event ev(buf, len); DBUG_EXECUTE_IF("crash_before_write_checkpoint_event", flush_io_cache(&log_file); @@ -3711,7 +3706,7 @@ bool MYSQL_BIN_LOG::open(const char *log_name, { WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::open(): Removing xid_list_entry for " "%s (%lu)", b); - my_free(binlog_xid_count_list.get()); + delete binlog_xid_count_list.get(); } mysql_cond_broadcast(&COND_xid_list); WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::open(): Adding new xid_list_entry for " @@ -3758,7 +3753,7 @@ Turning logging off for the whole duration of the MySQL server process. \ To turn it on again: fix the cause, \ shutdown the MySQL server and restart it.", name, errno); if (new_xid_list_entry) - my_free(new_xid_list_entry); + delete new_xid_list_entry; if (file >= 0) mysql_file_close(file, MYF(0)); close(LOG_CLOSE_INDEX); @@ -4252,7 +4247,7 @@ bool MYSQL_BIN_LOG::reset_logs(THD *thd, bool create_new_log, DBUG_ASSERT(b->xid_count == 0); WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::reset_logs(): Removing " "xid_list_entry for %s (%lu)", b); - my_free(binlog_xid_count_list.get()); + delete binlog_xid_count_list.get(); } mysql_cond_broadcast(&COND_xid_list); reset_master_pending--; @@ -9736,7 +9731,7 @@ TC_LOG_BINLOG::mark_xid_done(ulong binlog_id, bool write_checkpoint) break; WSREP_XID_LIST_ENTRY("TC_LOG_BINLOG::mark_xid_done(): Removing " "xid_list_entry for %s (%lu)", b); - my_free(binlog_xid_count_list.get()); + delete binlog_xid_count_list.get(); } mysql_mutex_unlock(&LOCK_xid_list); diff --git a/sql/log.h b/sql/log.h index b4c9b24a3a9..277e5c6f69c 100644 --- a/sql/log.h +++ b/sql/log.h @@ -587,7 +587,18 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG long xid_count; /* For linking in requests to the binlog background thread. */ xid_count_per_binlog *next_in_queue; - xid_count_per_binlog(); /* Give link error if constructor used. */ + xid_count_per_binlog(char *log_file_name, uint log_file_name_len) + :binlog_id(0), xid_count(0) + { + binlog_name_len= log_file_name_len; + binlog_name= (char *) my_malloc(binlog_name_len, MYF(MY_ZEROFILL)); + if (binlog_name) + memcpy(binlog_name, log_file_name, binlog_name_len); + } + ~xid_count_per_binlog() + { + my_free(binlog_name); + } }; I_List<xid_count_per_binlog> binlog_xid_count_list; mysql_mutex_t LOCK_binlog_background_thread;

1 0

d9716cfadb8: MDEV-20923:UBSAN: member access within address … which does not point to an object of type 'xid_count_per_binlog'
by sujatha 23 Jan '20

23 Jan '20

revision-id: d9716cfadb8cd3448f25d8b411451f87064cecaf (mariadb-10.1.43-66-gd9716cfadb8) parent(s): 982294ac1680938ac9223fb64a64e21f0cbc322a author: Sujatha committer: Sujatha timestamp: 2020-01-23 16:00:41 +0530 message: MDEV-20923:UBSAN: member access within address … which does not point to an object of type 'xid_count_per_binlog' Problem: ------- Accessing a member within 'xid_count_per_binlog' structure results in following error when 'UBSAN' is enabled. member access within address 0xXXX which does not point to an object of type 'xid_count_per_binlog' Analysis: --------- The problem appears to be that no constructor for 'xid_count_per_binlog' is being called, and thus the vtable will not be initialized. Fix: --- Defined a parameterized constructor for 'xid_count_per_binlog' class. --- sql/log.cc | 27 +++++++++++---------------- sql/log.h | 14 +++++++++++++- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/sql/log.cc b/sql/log.cc index acf1f8f8a9c..0efef6d1e29 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -3216,7 +3216,7 @@ void MYSQL_BIN_LOG::cleanup() DBUG_ASSERT(!binlog_xid_count_list.head()); WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::cleanup(): Removing xid_list_entry " "for %s (%lu)", b); - my_free(b); + delete b; } mysql_mutex_destroy(&LOCK_log); @@ -3580,17 +3580,9 @@ bool MYSQL_BIN_LOG::open(const char *log_name, */ uint off= dirname_length(log_file_name); uint len= strlen(log_file_name) - off; - char *entry_mem, *name_mem; - if (!(new_xid_list_entry = (xid_count_per_binlog *) - my_multi_malloc(MYF(MY_WME), - &entry_mem, sizeof(xid_count_per_binlog), - &name_mem, len, - NULL))) + new_xid_list_entry= new xid_count_per_binlog(log_file_name+off, (int)len); + if (!new_xid_list_entry) goto err; - memcpy(name_mem, log_file_name+off, len); - new_xid_list_entry->binlog_name= name_mem; - new_xid_list_entry->binlog_name_len= len; - new_xid_list_entry->xid_count= 0; /* Find the name for the Initial binlog checkpoint. @@ -3607,7 +3599,10 @@ bool MYSQL_BIN_LOG::open(const char *log_name, mysql_mutex_unlock(&LOCK_xid_list); if (!b) b= new_xid_list_entry; - strmake(buf, b->binlog_name, b->binlog_name_len); + if (b->binlog_name) + strmake(buf, b->binlog_name, b->binlog_name_len); + else + goto err; Binlog_checkpoint_log_event ev(buf, len); DBUG_EXECUTE_IF("crash_before_write_checkpoint_event", flush_io_cache(&log_file); @@ -3711,7 +3706,7 @@ bool MYSQL_BIN_LOG::open(const char *log_name, { WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::open(): Removing xid_list_entry for " "%s (%lu)", b); - my_free(binlog_xid_count_list.get()); + delete binlog_xid_count_list.get(); } mysql_cond_broadcast(&COND_xid_list); WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::open(): Adding new xid_list_entry for " @@ -3758,7 +3753,7 @@ Turning logging off for the whole duration of the MySQL server process. \ To turn it on again: fix the cause, \ shutdown the MySQL server and restart it.", name, errno); if (new_xid_list_entry) - my_free(new_xid_list_entry); + delete new_xid_list_entry; if (file >= 0) mysql_file_close(file, MYF(0)); close(LOG_CLOSE_INDEX); @@ -4252,7 +4247,7 @@ bool MYSQL_BIN_LOG::reset_logs(THD *thd, bool create_new_log, DBUG_ASSERT(b->xid_count == 0); WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::reset_logs(): Removing " "xid_list_entry for %s (%lu)", b); - my_free(binlog_xid_count_list.get()); + delete binlog_xid_count_list.get(); } mysql_cond_broadcast(&COND_xid_list); reset_master_pending--; @@ -9736,7 +9731,7 @@ TC_LOG_BINLOG::mark_xid_done(ulong binlog_id, bool write_checkpoint) break; WSREP_XID_LIST_ENTRY("TC_LOG_BINLOG::mark_xid_done(): Removing " "xid_list_entry for %s (%lu)", b); - my_free(binlog_xid_count_list.get()); + delete binlog_xid_count_list.get(); } mysql_mutex_unlock(&LOCK_xid_list); diff --git a/sql/log.h b/sql/log.h index b4c9b24a3a9..e69c9c39eaf 100644 --- a/sql/log.h +++ b/sql/log.h @@ -587,7 +587,19 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG long xid_count; /* For linking in requests to the binlog background thread. */ xid_count_per_binlog *next_in_queue; - xid_count_per_binlog(); /* Give link error if constructor used. */ + xid_count_per_binlog(char *log_file_name, uint log_file_name_len) + :binlog_name(log_file_name), binlog_name_len(log_file_name_len), + binlog_id(0), xid_count(0) + { + binlog_name_len= log_file_name_len; + binlog_name= (char *) my_malloc(binlog_name_len, MYF(MY_ZEROFILL)); + if (binlog_name) + memcpy(binlog_name, log_file_name, binlog_name_len); + } + ~xid_count_per_binlog() + { + my_free(binlog_name); + } }; I_List<xid_count_per_binlog> binlog_xid_count_list; mysql_mutex_t LOCK_binlog_background_thread;

1 0

[Maria-developers] 43f27ae4778: MDEV-21490: binlog tests fail with valgrind: Conditional jump or move depends on uninitialised value in sql_ex_info::init
by sujatha 23 Jan '20

23 Jan '20

revision-id: 43f27ae4778b3e7be86897e49c182dc05080fd88 (mariadb-10.1.43-66-g43f27ae4778) parent(s): 982294ac1680938ac9223fb64a64e21f0cbc322a author: Sujatha committer: Sujatha timestamp: 2020-01-23 11:57:46 +0530 message: MDEV-21490: binlog tests fail with valgrind: Conditional jump or move depends on uninitialised value in sql_ex_info::init Problem: ======= P1) Conditional jump or move depends on uninitialised value(s) sql_ex_info::init(char const*, char const*, bool) (log_event.cc:3083) code: All the following variables are not initialized. ---- return ((cached_new_format != -1) ? cached_new_format : (cached_new_format=(field_term_len > 1 || enclosed_len > 1 || line_term_len > 1 || line_start_len > 1 || escaped_len > 1))); P2) Conditional jump or move depends on uninitialised value(s) Rows_log_event::Rows_log_event(char const*, unsigned int, Format_description_log_event const*) (log_event.cc:9571) Code: Uninitialized values is reported for 'var_header_len' variable. ---- if (var_header_len < 2 || event_len < static_cast<unsigned int>(var_header_len + (post_start - buf))) P3) Conditional jump or move depends on uninitialised value(s) Table_map_log_event::pack_info(Protocol*) (log_event.cc:11553) code:'m_table_id' is uninitialized. ---- void Table_map_log_event::pack_info(Protocol *protocol) ... size_t bytes= my_snprintf(buf, sizeof(buf), "table_id: %lu (%s.%s)", m_table_id, m_dbnam, m_tblnam); Fix: === P1 - Fix) Initialize cached_new_format,field_term_len, enclosed_len, line_term_len, line_start_len, escaped_len members in default constructor. P2 - Fix) "var_header_len" is initialized by reading the event buffer. In case of an invalid event the buffer will contain invalid data. Hence added a check to validate the event data. If event_len is smaller than valid header length return immediately. P3 - Fix) 'm_table_id' within Table_map_log_event is initialized by reading data from the event buffer. Use 'VALIDATE_BYTES_READ' macro to validate the current state of the buffer. If it is invalid return immediately. --- sql/log_event.cc | 7 +++++++ sql/log_event.h | 9 ++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/sql/log_event.cc b/sql/log_event.cc index e8881c77f2b..9924b9a0493 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -9535,6 +9535,12 @@ Rows_log_event::Rows_log_event(const char *buf, uint event_len, uint8 const post_header_len= description_event->post_header_len[event_type-1]; + if (event_len < (uint)(common_header_len + post_header_len)) + { + m_cols.bitmap= 0; + DBUG_VOID_RETURN; + } + DBUG_PRINT("enter",("event_len: %u common_header_len: %d " "post_header_len: %d", event_len, common_header_len, @@ -11043,6 +11049,7 @@ Table_map_log_event::Table_map_log_event(const char *buf, uint event_len, const char *post_start= buf + common_header_len; post_start+= TM_MAPID_OFFSET; + VALIDATE_BYTES_READ(post_start, buf, event_len); if (post_header_len == 6) { /* Master is of an intermediate source tree before 5.1.4. Id is 4 bytes */ diff --git a/sql/log_event.h b/sql/log_event.h index 2c8dc3d7353..04fce65faeb 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -2057,7 +2057,14 @@ class Query_log_event: public Log_event ****************************************************************************/ struct sql_ex_info { - sql_ex_info() {} /* Remove gcc warning */ + sql_ex_info(): + cached_new_format(-1), + field_term_len(0), + enclosed_len(0), + line_term_len(0), + line_start_len(0), + escaped_len(0) + {} /* Remove gcc warning */ const char* field_term; const char* enclosed; const char* line_term;

1 0

[Maria-developers] A cleanup for libmariadb
by Alexander Barkov 20 Jan '20

20 Jan '20

Hi Georg, Can you please review this cleanup patch? It's a prerequisite for: MDEV-17832 Protocol: extensions for Pluggable types and JSON, GEOMETRY The idea is that this patch introduces a new file libmariadb/mariadb_priv.h and adds prototypes of these functions: void free_rows(MYSQL_DATA *cur); int ma_multi_command(MYSQL *mysql, enum enum_multi_status status); MYSQL_FIELD * unpack_fields(MYSQL_DATA *data, MA_MEM_ROOT *alloc,uint fields, my_bool default_value); and removes these prototypes from mariadb_stmt.c. I think it should be safer this way. Later I'll add there more prototypes into libmariadb/mariadb_priv.h, e.g. functions that will be added under terms of MDEV-17832. Note, I also removed the "my_bool long_flag_protocol" parameter from unpack_fields(), it's not used anyway. Under terms of MDEV-17832 I'll have to add a "const MYSQL *mysql" parameter. So if we ever need to check CLIENT_LONG_FLAG inside unpack_fields() again, it can be done through the "mysql" parameter: (mysql->server_capabilities & CLIENT_LONG_FLAG) Thanks!

1 0

[Maria-developers] A cleanup for libmariadb
by Alexander Barkov 20 Jan '20

20 Jan '20

Hi Georg, Can you please review this cleanup patch? It's a prerequisite for: MDEV-17832 Protocol: extensions for Pluggable types and JSON, GEOMETRY The idea is that this patch introduces a new file libmariadb/mariadb_priv.h and adds prototypes of these functions: void free_rows(MYSQL_DATA *cur); int ma_multi_command(MYSQL *mysql, enum enum_multi_status status); MYSQL_FIELD * unpack_fields(MYSQL_DATA *data, MA_MEM_ROOT *alloc,uint fields, my_bool default_value); and removes these prototypes from mariadb_stmt.c. I think it should be safer this way. Later I'll add there more prototypes into libmariadb/mariadb_priv.h, e.g. functions that will be added under terms of MDEV-17832. Note, I also removed the "my_bool long_flag_protocol" parameter from unpack_fields(), it's not used anyway. Under terms of MDEV-17832 I'll have to add a "const MYSQL *mysql" parameter. So if we ever need to check CLIENT_LONG_FLAG inside unpack_fields() again, it can be done through the "mysql" parameter: (mysql->server_capabilities & CLIENT_LONG_FLAG) Thanks!

1 0

Re: [Maria-developers] 9d2c63d45e4: MDEV-20076: SHOW GRANTS does not quote role names properly
by Sergei Golubchik 19 Jan '20

19 Jan '20

Hi, Oleksandr! On Jan 19, Oleksandr Byelkin wrote: > revision-id: 9d2c63d45e4 (mariadb-10.3.20-6-g9d2c63d45e4) > parent(s): d4edb0510ec > author: Oleksandr Byelkin <sanja(a)mariadb.com> > committer: Oleksandr Byelkin <sanja(a)mariadb.com> > timestamp: 2019-11-14 09:32:54 +0100 > message: > > MDEV-20076: SHOW GRANTS does not quote role names properly > > Quotes added to output. > > diff --git a/mysql-test/main/grant5.test b/mysql-test/main/grant5.test > index 649bba7d1ca..045cbf8fc86 100644 > --- a/mysql-test/main/grant5.test > +++ b/mysql-test/main/grant5.test > @@ -33,3 +33,26 @@ REVOKE EXECUTE ON PROCEDURE sp FROM u; > --error ER_TABLE_NOT_LOCKED > REVOKE PROCESS ON *.* FROM u; > DROP TABLE t1; > + > +--echo # > +--echo # MDEV-20076: SHOW GRANTS does not quote role names properly > +--echo # > + > +create role 'role-1'; > +create role 'rock\'n\'roll'; > +create user 'user-1'@'localhost'; > +create user 'O\'Brien'@'localhost'; > +grant select on mysql.user to 'role-1'; > +grant select on mysql.user to 'rock\'n\'roll'; > +GRANT 'role-1' TO 'user-1'@'localhost'; > +GRANT 'rock\'n\'roll' TO 'O\'Brien'@'localhost'; > +show grants for 'role-1'; > +show grants for 'rock\'n\'roll'; > +show grants for 'user-1'@'localhost'; > +show grants for 'O\'Brien'@'localhost'; > +drop role 'role-1'; > +drop role 'rock\'n\'roll'; > +drop user 'user-1'@'localhost'; > +drop user 'O\'Brien'@'localhost'; > + > +--echo # End of 10.3 tests please add few tests when sql_quote_show_create=0. Like 1. sql_quote_show_create=0, role name is not reserved (e.g. foo) 2. sql_quote_show_create=0, role name is reserved (e.g. fetch) 3. sql_quote_show_create=0, role name is not a valid identifier (e.g. role-1) Then ok to push. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] edc9059c31b: MDEV-18027: Running out of file descriptors and eventual crash
by Sergei Golubchik 19 Jan '20

19 Jan '20

Hi, Oleksandr! Ok to push. Just one comment, see below: On Jan 19, Oleksandr Byelkin wrote: > revision-id: edc9059c31b (mariadb-10.2.28-3-gedc9059c31b) > parent(s): 0b1bc4bf76f > author: Oleksandr Byelkin <sanja(a)mariadb.com> > committer: Oleksandr Byelkin <sanja(a)mariadb.com> > timestamp: 2019-11-05 16:24:26 +0100 > message: > > MDEV-18027: Running out of file descriptors and eventual crash > > For automatic number of opened files limit take into account number of > table instances for table cache > > diff --git a/sql/mysqld.cc b/sql/mysqld.cc > index 34e5704bcfe..9907c17ee07 100644 > --- a/sql/mysqld.cc > +++ b/sql/mysqld.cc > @@ -4461,6 +4461,17 @@ static int init_common_variables() > if (files < wanted_files && global_system_variables.log_warnings) > sql_print_warning("Could not increase number of max_open_files to more than %u (request: %u)", files, wanted_files); > > + /* > + If we required too much tc_instances than we reduce it but not more > + then in half The comment is wrong, you don't do "not more than in half", you can reduce it all the way to 1. Which, I think, it correct. So the code is ok, but the comment is misleading, please remove the "but not more than in half" part. > + */ > + SYSVAR_AUTOSIZE_IF_CHANGED(tc_instances, > + (uint32) MY_MIN(MY_MAX((files - extra_files - > + max_connections)/ > + 2/tc_size, > + 1), > + tc_instances), > + uint32); > /* > If we have requested too much file handles than we bring > max_connections in supported bounds. Still leave at least Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

[Maria-developers] 回复: [Maria-discuss] [MariaDB 10.3.7] Hello, everyone. My process core dumped while calling mysql_ping().
by Ma Allen 15 Jan '20

15 Jan '20

Hi Markus, Only MySqlWaitConnected() in the two functions doesn't use lock, reading database uses mutex. What's more, the two functions in initialization relates to ONLY the main thread. No other threads call them. Previously, the program is written with mysqllib, which is not MariaDB lib currently used. Mysqllib supports multiple threads to concurrently use one connection. Its tutorials were also referred to: https://dev.mysql.com/doc/refman/5.7/en/c-api-threaded-clients.html MySQL :: MySQL 5.7 Reference Manual :: 27.7.3.3 Writing C API Threaded Client Programs<https://dev.mysql.com/doc/refman/5.7/en/c-api-threaded-clients.html> This section provides guidance for writing client programs that use the thread-related functions in the MySQL C API. For further information about these functions, see Section 27.7.11, “C API Threaded Function Descriptions”.For examples of source code that uses them, look in the client directory of a MySQL source distribution: dev.mysql.com After replacing mysqllib with MariaDB lib, everything works well on the test environment except one of the clients'. Allen ________________________________ 发件人: Maria-discuss <maria-discuss-bounces+mazhh=outlook.com(a)lists.launchpad.net> 代表 Markus Mäkelä <markus.makela(a)mariadb.com> 发送时间: 2020年1月14日 15:40 收件人: maria-discuss(a)lists.launchpad.net <maria-discuss(a)lists.launchpad.net> 主题: Re: [Maria-discuss] [Maria-developers] [MariaDB 10.3.7] Hello, everyone. My process core dumped while calling mysql_ping(). Hi, Is the connection protected by a lock of some sorts? To my knowledge the library does not support concurrent use of a single connection. I'd recommend trying to first wrap all access to the connection with a mutex and see if that solves the problem. Markus On 1/14/20 08:33, Ma Allen wrote: Environment: MariaDB 10.3.7 database server & related headers and libraries CentOS 7.3 or 7.4 ./configure --prefix=/home/spiderflow --localstatedir=/home/spiderflow --enable-unix-socket --with-libnss-libraries=/usr/lib64 --with-libnss-includes=/usr/include/nss3 --with-libnspr-libraries=/usr/lib64 --with-libnspr-includes=/usr/include/nspr4 --enable-netmap --enable-rust --disable-gccmarch-native --enable-mysql --with-libmysql-includes=/usr/local/mariadb-10.3.7-linux-systemd-x86_64/include/mysql --with-libmysql-libraries=/usr/local/mariadb-10.3.7-linux-systemd-x86_64/lib Phenomenon: (gdb) #0 ma_pvio_write (pvio=0x10101010101b2e0, buffer=buffer@entry=0x2510101 "\001", length=length@entry=5) at /home/buildbot/buildbot/build/libmariadb/libmariadb/ma_pvio.c:350 #1 0x00007f7100249dea in ma_net_real_write (net=net@entry=0xee9de0 <g_mysql_info>, packet=0x2510101 "\001", len=5) at /home/buildbot/buildbot/build/libmariadb/libmariadb/ma_net.c:335 #2 0x00007f7100249f60 in ma_net_flush (net=net@entry=0xee9de0 <g_mysql_info>) at /home/buildbot/buildbot/build/libmariadb/libmariadb/ma_net.c:166 #3 0x00007f710024a71a in ma_net_write_command (net=net@entry=0xee9de0 <g_mysql_info>, command=command@entry=14 '\016', packet=packet@entry=0x7f71003d6c67 "", len=0, disable_flush=disable_flush@entry=0 '\000') at /home/buildbot/buildbot/build/libmariadb/libmariadb/ma_net.c:244 #4 0x00007f7100252ca4 in mthd_my_send_cmd (mysql=0xee9de0 <g_mysql_info>, command=<optimized out>, arg=0x7f71003d6c67 "", length=0, skipp_check=<optimized out>, opt_arg=<optimized out>) at /home/buildbot/buildbot/build/libmariadb/libmariadb/mariadb_lib.c:394 #5 0x00007f71002518d0 in mysql_ping (mysql=0xee9de0 <g_mysql_info>) at /home/buildbot/buildbot/build/libmariadb/libmariadb/mariadb_lib.c:2552 #6 0x00000000006651b2 in MySqlWaitConnected (p_mysql_data=p_mysql_data@entry=0xee9de0 <g_mysql_info>) at ../srcSF/spiderFlow-proto-baseline.c:676 #7 0x0000000000668819 in LoadProtoConfInfos (pInfo=pInfo@entry=0xee9de0 <g_mysql_info>) at ../srcSF/spiderFlow-proto-baseline.c:835 #8 0x000000000066c53a in InitBaseLine () at ../srcSF/spiderFlow-proto-baseline.c:1069 #9 0x00000000005f308b in PostConfLoadedSetup (suri=0x10dbb80 <suricata>) at suricata.c:2900 #10 0x0000000000425ce8 in main (argc=5, argv=0x7fff1cdca258) at suricata.c:3072 (gdb) Analyzation: The core dump always exists after running for a while or maybe after the sever restarts. It occurs at the initialization and involves with ONLY ONE thread, which is the main thread of the whole process. The related code is two functions, which both checks the connection and read data from MariaDB. The first function works fine, however, the second one's checking connection leads to core dump. The code to check connection is as follows: void MySqlWaitConnected(MYSQL * p_mysql_data) { while (mysql_ping(p_mysql_data) != 0) { SCLogInfo("Mysql ping error:%s", mysql_error(p_mysql_data)); #ifdef WIN32 Sleep(3000); #else sleep(3); #endif } } Previously, I solved a likely similar problem as follows: Multiple threads shared ONE MySQL handler, which is the MYSQL data structure. Before inserting data into MariaDB without mutex, MySqlWaitConnected() is called. So while multiple threads do the same operation, inserting data is interrupted by mysql_ping() that leads to core dump. Afterwards, I comment MySqlWaitConnected() and add mutex for mysql_real_query() and mysql_commit(). Does anyone meet the same problem before? Any suggestion is appreciated. Thanks in advance. Best Regards, Allen Ma _______________________________________________ Mailing list: https://launchpad.net/~maria-developers Post to : maria-developers(a)lists.launchpad.net<mailto:maria-developers@lists.launchpad.net> Unsubscribe : https://launchpad.net/~maria-developers More help : https://help.launchpad.net/ListHelp -- Markus Mäkelä, Senior Software Engineer MariaDB Corporation t: +358 40 7740484 | IRC: markusjm@freenode

1 2

Re: [Maria-developers] 10670133744: MDEV-16978 Application-time periods: WITHOUT OVERLAPS
by Sergei Golubchik 14 Jan '20

14 Jan '20

Hi, Nikita! On Dec 16, Nikita Malyavin wrote: > revision-id: 10670133744 (mariadb-10.4.4-492-g10670133744) > parent(s): 251c6e17269 > author: Nikita Malyavin <nikitamalyavin(a)gmail.com> > committer: Nikita Malyavin <nikitamalyavin(a)gmail.com> > timestamp: 2019-11-26 19:22:04 +1000 > message: > > MDEV-16978 Application-time periods: WITHOUT OVERLAPS > > * The overlaps check is implemented on a handler level per row command. It creates a separate cursor (actually, another handler instance) and caches it inside the original handler, when ha_update_row or ha_insert_row is issued. Cursor closes on unlocking the handler. > > * Containing the same key in index means unique constraint violation even in usual terms. So we fetch left and right neighbours and check that they have same key prefix, excluding from the key only the period part. If it doesnt match, then there's no such neighbour, and the check passes. Otherwise, we check if this neighbour intersects with the considered key. > > * the check does introduce new error and fails with ER_DUPP_KEY error. This might break REPLACE workflow and should be fixed separately wrap long lines, pease. > diff --git a/sql/handler.cc b/sql/handler.cc > index 46a0c313c80..ed36f3c5bd3 100644 > --- a/sql/handler.cc > +++ b/sql/handler.cc > @@ -6402,6 +6407,14 @@ int handler::ha_external_lock(THD *thd, int lock_type) > mysql_audit_external_lock(thd, table_share, lock_type); > } > > + if (lock_type == F_UNLCK && check_overlaps_handler) > + { > + check_overlaps_handler->ha_external_lock(table->in_use, F_UNLCK); > + check_overlaps_handler->close(); > + check_overlaps_handler= NULL; > + overlaps_error_key= -1; > + } I'm still thinking about how avoid this overhead or at least to simplify the code. One option is to use HA_EXTRA_REMEMBER_POS It doesn't nest, you're right, but it can be fixed. Another, simpler, option is to use TABLE::update_handler. This is a second auxillary handler, a clone, exactly as yours, created for to check long uniques. So I don't see a need to create a yet another clone when you can simply use TABLE::update_handler. It's never used for scanning, only for point lookups, so there is no position that you can disrupt. upd: you seem to have got the same idea. and you're right, it should be in the handler class, not in the TABLE as I originally wanted. > + > if (MYSQL_HANDLER_RDLOCK_DONE_ENABLED() || > MYSQL_HANDLER_WRLOCK_DONE_ENABLED() || > MYSQL_HANDLER_UNLOCK_DONE_ENABLED()) > @@ -6935,6 +6956,134 @@ void handler::set_lock_type(enum thr_lock_type lock) > table->reginfo.lock_type= lock; > } > > +int handler::ha_check_overlaps(const uchar *old_data, const uchar* new_data) > +{ > + DBUG_ASSERT(new_data); > + if (!table_share->period.unique_keys) > + return 0; > + if (table->versioned() && !table->vers_end_field()->is_max()) > + return 0; > + > + bool is_update= old_data != NULL; > + if (!check_overlaps_buffer) > + check_overlaps_buffer= (uchar*)alloc_root(&table_share->mem_root, > + table_share->max_unique_length > + + table_share->reclength); > + auto *record_buffer= check_overlaps_buffer + table_share->max_unique_length; > + auto *handler= this; Please, add a comment that on INSERT handler->inited can be NONE > + if (handler->inited != NONE) > + { > + if (!check_overlaps_handler) > + { > + check_overlaps_handler= clone(table_share->normalized_path.str, > + &table_share->mem_root); > + int error= -1; > + if (check_overlaps_handler != NULL) > + error= check_overlaps_handler->ha_external_lock(table->in_use, F_RDLCK); > + if (error) > + return error; > + } > + handler= check_overlaps_handler; > + > + // Needs to compare record refs later is old_row_found() > + if (is_update) > + position(old_data); > + } > + > + // Save and later restore this handler's keyread > + int old_this_keyread= this->keyread; what's that for? you aren't using `this` anywhere below. Unless handler == this, but then this->keyread cannot be enabled. > + DBUG_ASSERT(this->ha_end_keyread() == 0); Eh. Never put any code with side effects into an assert. Assert are conditionally compiled. > + > + int error= 0; > + > + for (uint key_nr= 0; key_nr < table_share->keys && !error; key_nr++) > + { > + const KEY &key_info= table->key_info[key_nr]; > + const uint key_parts= key_info.user_defined_key_parts; > + if (!key_info.without_overlaps) > + continue; > + > + if (is_update) > + { > + bool key_used= false; > + for (uint k= 0; k < key_parts && !key_used; k++) > + key_used= bitmap_is_set(table->write_set, > + key_info.key_part[k].fieldnr - 1); > + if (!key_used) > + continue; > + } > + > + error= handler->ha_index_init(key_nr, 0); > + if (error) > + return error; > + > + auto old_row_found= [is_update, old_data, record_buffer, this, handler](){ There is no reason to use a lambda here. could you rewrite that, please? In this particular case old_row_found() is only used once, so you can inline your lambda there. > + if (!is_update) > + return false; > + /* In case of update it could appear that the nearest neighbour is > + * a record we are updating. It means, that there are no overlaps > + * from this side. > + * > + * An assumption is made that during update we always have the last > + * fetched row in old_data. Therefore, comparing ref's is enough > + * */ > + DBUG_ASSERT(handler != this && inited != NONE); as a general rule please use DBUG_ASSERT(x); DBUG_ASSERT(y); and not DBUG_ASSERT(x && y); > + DBUG_ASSERT(ref_length == handler->ref_length); > + > + handler->position(record_buffer); > + return memcmp(ref, handler->ref, ref_length) == 0; > + }; > + > + error= handler->ha_start_keyread(key_nr); > + DBUG_ASSERT(!error); > + > + const uint period_field_length= key_info.key_part[key_parts - 1].length; > + const uint key_base_length= key_info.key_length - 2 * period_field_length; > + > + key_copy(check_overlaps_buffer, new_data, &key_info, 0); > + > + /* Copy period_end to period_start. > + * the value in period_end field is not significant, but anyway let's leave > + * it defined to avoid uninitialized memory access > + */ please format your multi-line comments to follow the existing server code conventions > + memcpy(check_overlaps_buffer + key_base_length, > + check_overlaps_buffer + key_base_length + period_field_length, > + period_field_length); > + > + /* Find row with period_start < (period_end of new_data) */ > + error = handler->ha_index_read_map(record_buffer, > + check_overlaps_buffer, > + key_part_map((1 << key_parts) - 1), > + HA_READ_BEFORE_KEY); > + > + if (!error && old_row_found()) > + error= handler->ha_index_prev(record_buffer); > + > + if (!error && table->check_period_overlaps(key_info, key_info, > + new_data, record_buffer) == 0) > + error= HA_ERR_FOUND_DUPP_KEY; > + > + if (error == HA_ERR_KEY_NOT_FOUND || error == HA_ERR_END_OF_FILE) > + error= 0; > + > + if (error == HA_ERR_FOUND_DUPP_KEY) > + overlaps_error_key= key_nr; > + > + int end_error= handler->ha_end_keyread(); > + DBUG_ASSERT(!end_error); > + > + end_error= handler->ha_index_end(); > + if (!error && end_error) > + error= end_error; > + } > + > + // Restore keyread of this handler, if it was enabled > + if (old_this_keyread < MAX_KEY) > + DBUG_ASSERT(this->ha_start_keyread(old_this_keyread) == 0); > + > + return error; > +} > + > #ifdef WITH_WSREP > /** > @details > diff --git a/sql/key.cc b/sql/key.cc > index bf50094a9e4..d4f33467e2b 100644 > --- a/sql/key.cc > +++ b/sql/key.cc > @@ -899,3 +899,45 @@ bool key_buf_cmp(KEY *key_info, uint used_key_parts, > } > return FALSE; > } > + > + a comment please. What does the function return? -1/0/1 ? > +int key_period_compare_bases(const KEY &lhs_key, const KEY &rhs_key, > + const uchar *lhs, const uchar *rhs) > +{ > + uint base_part_nr= lhs_key.user_defined_key_parts - 2; > + int cmp_res= 0; > + for (uint part_nr= 0; !cmp_res && part_nr < base_part_nr; part_nr++) > + { > + Field *f= lhs_key.key_part[part_nr].field; > + cmp_res= f->cmp(f->ptr_in_record(lhs), > + rhs_key.key_part[part_nr].field->ptr_in_record(rhs)); > + } > + > + return cmp_res; > +} > + a comment please. What does the function return? > +int key_period_compare_periods(const KEY &lhs_key, const KEY &rhs_key, > + const uchar *lhs, const uchar *rhs) > +{ > + uint base_part_nr= lhs_key.user_defined_key_parts - 2; > + > + Field *lhs_fields[]= {lhs_key.key_part[base_part_nr].field, > + lhs_key.key_part[base_part_nr + 1].field}; > + > + Field *rhs_fields[]= {rhs_key.key_part[base_part_nr].field, > + rhs_key.key_part[base_part_nr + 1].field}; > + > + int cmp[2][2]; /* l1 > l2, l1 > r2, r1 > l2, r1 > r2 */ > + for (int i= 0; i < 2; i++) > + { > + for (int j= 0; j < 2; j++) > + { > + cmp[i][j]= lhs_fields[0]->cmp(lhs_fields[i]->ptr_in_record(lhs), > + rhs_fields[j]->ptr_in_record(rhs)); > + } > + } > + > + bool overlaps = (cmp[0][0] <= 0 && cmp[1][0] > 0) > + || (cmp[0][0] >= 0 && cmp[0][1] < 0); > + return overlaps ? 0 : cmp[0][0]; Couldn't this be simplifed? Like if (cmp[1][0] <= 0) // r1 <= l2 return -1; if (cmp[0][1] >= 0) // l1 >= r2 return 1; return 0; and I think it'd be clearer to remove this cmp[][] array and write the condition directly. May be with shortcuts like const Field const * f=lhs_fields[0]; const uchar const * l1=lhs_fields[0]->ptr_in_record(lhs), ... if (f->cmp(r1, l2) <= 0) return -1; if (f->cmp(l1, r2) >= 0) return 1; return 0; > +} > \ No newline at end of file > diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt > index 36188e58624..bb2f0fc5296 100644 > --- a/sql/share/errmsg-utf8.txt > +++ b/sql/share/errmsg-utf8.txt > @@ -7943,3 +7943,9 @@ ER_WARN_HISTORY_ROW_START_TIME > eng "Table `%s.%s` history row start '%s' is later than row end '%s'" > ER_PART_STARTS_BEYOND_INTERVAL > eng "%`s: STARTS is later than query time, first history partition may exceed INTERVAL value" > +ER_KEY_CONTAINS_PERIOD_FIELDS > + eng "Key %`s should not contain period fields in order to make it WITHOUT OVERLAPS" I'd say "Key %`s cannot explicitly include column %`s" or "WITHOUT OVERLAPS key %`s cannot explicitly include column %`s" but I think the latter looks weird, I like the first variant more > +ER_PERIOD_WITHOUT_OVERLAPS_PARTITIONED > + eng "Period WITHOUT OVERLAPS is not implemented for partitioned tables" I don't think we should create a separate error message for every feature that doesn't work with partitioning. There's already ER_FOREIGN_KEY_ON_PARTITIONED eng "Foreign key clause is not yet supported in conjunction with partitioning" ER_PARTITION_NO_TEMPORARY eng "Cannot create temporary table with partitions" ER_FULLTEXT_NOT_SUPPORTED_WITH_PARTITIONING eng "FULLTEXT index is not supported for partitioned tables" which is two error messages too many. So I'd just say ER_FEATURE_NOT_SUPPORTED_WITH_PARTITIONING end "Partitioned tables do not support %s" where %s could be CREATE TEMPORARY TABLE, FOREIGN KEY, FULLTEXT, WITHOUT OVERLAPS, and whatever else partitioned tables don't or won't support. Note that you cannot remove old error messages, so just rename the first one to ER_FEATURE_NOT_SUPPORTED_WITH_PARTITIONING, and keep the rest unused but still present in the errmsg-utf8.txt file. > +ER_PERIOD_WITHOUT_OVERLAPS_NON_UNIQUE > + eng "Period WITHOUT OVERLAPS is only allowed for unique keys" "Only UNIQUE or PRIMARY keys can be WITHOUT OVERLAPS" > diff --git a/sql/table.cc b/sql/table.cc > index 7ed5121a9c6..8b84fb3035d 100644 > --- a/sql/table.cc > +++ b/sql/table.cc > @@ -1519,6 +1520,15 @@ bool read_extra2(const uchar *frm_image, size_t len, extra2_fields *fields) > size_t length= extra2_read_len(&extra2, e2end); > if (!length) > DBUG_RETURN(true); > + > + auto fill_extra2= [extra2, length](LEX_CUSTRING *section){ > + if (section->str) > + return true; > + *section= {extra2, length}; > + return false; > + }; don't use a lambda here either, make it a proper function > + > + bool fail= false; > switch (type) { > case EXTRA2_TABLEDEF_VERSION: > if (fields->version.str) // see init_from_sql_statement_string() > @@ -1726,11 +1725,26 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, > keyinfo= &first_keyinfo; > thd->mem_root= &share->mem_root; > > + auto err= [thd, share, &handler_file, &se_plugin, old_root](){ > + share->db_plugin= NULL; > + share->error= OPEN_FRM_CORRUPTED; > + share->open_errno= my_errno; > + delete handler_file; > + plugin_unlock(0, se_plugin); > + my_hash_free(&share->name_hash); > + > + if (!thd->is_error()) > + open_table_error(share, OPEN_FRM_CORRUPTED, share->open_errno); > + > + thd->mem_root= old_root; > + return HA_ERR_NOT_A_TABLE; > + }; Revert that too > + > if (write && write_frm_image(frm_image, frm_length)) > - goto err; > + DBUG_RETURN(err()); > > if (frm_length < FRM_HEADER_SIZE + FRM_FORMINFO_SIZE) > - goto err; > + DBUG_RETURN(err()); > > share->frm_version= frm_image[2]; > /* > @@ -2251,14 +2265,30 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, > pos+= period.constr_name.length; > > if (init_period_from_extra2(&period, pos, end)) > - goto err; > + DBUG_RETURN(err()); > status_var_increment(thd->status_var.feature_application_time_periods); > } > > + if (extra2.without_overlaps.str) > + { > + const uchar *key_pos= extra2.without_overlaps.str; > + period.unique_keys= read_frm_keyno(key_pos); > + for (uint k= 0; k < period.unique_keys; k++) > + { > + key_pos+= frm_keyno_size; > + uint key_nr= read_frm_keyno(key_pos); > + key_info[key_nr].without_overlaps= true; > + } > + > + if ((period.unique_keys + 1) * frm_keyno_size > + != extra2.without_overlaps.length) > + DBUG_RETURN(err()); you can also check here that extra2.application_period.str != NULL otherwise it's OPEN_FRM_CORRUPTED too > + } > + > if (extra2.field_data_type_info.length && > field_data_type_info_array.parse(old_root, share->fields, > extra2.field_data_type_info)) > - goto err; > + DBUG_RETURN(err()); > > for (i=0 ; i < share->fields; i++, strpos+=field_pack_length, field_ptr++) > { > @@ -8600,6 +8616,15 @@ void TABLE::evaluate_update_default_function() > DBUG_VOID_RETURN; > } > a comment please. What does the function return? > +int TABLE::check_period_overlaps(const KEY &lhs_key, const KEY &rhs_key, > + const uchar *lhs, const uchar *rhs) > +{ > + int cmp_res= key_period_compare_bases(lhs_key, rhs_key, lhs, rhs); > + if (cmp_res) > + return cmp_res; > + > + return key_period_compare_periods(lhs_key, rhs_key, lhs, rhs); > +} > > void TABLE::vers_update_fields() > { > diff --git a/sql/table.h b/sql/table.h > index 043341db608..e871471101f 100644 > --- a/sql/table.h > +++ b/sql/table.h > @@ -1620,6 +1621,13 @@ struct TABLE > int period_make_insert(Item *src, Field *dst); > int insert_portion_of_time(THD *thd, const vers_select_conds_t &period_conds, > ha_rows *rows_inserted); > + /* > + @return -1, lhs precedes rhs > + 0, lhs overlaps rhs > + 1, lhs succeeds rhs > + */ ah, a comment, good :) but move it to the function definition, please > + static int check_period_overlaps(const KEY &lhs_key, const KEY &rhs_key, > + const uchar *lhs, const uchar *rhs); > int delete_row(); > void vers_update_fields(); > void vers_update_end(); > @@ -1759,10 +1767,18 @@ class IS_table_read_plan; > > /** number of bytes used by field positional indexes in frm */ > constexpr uint frm_fieldno_size= 2; > +/** number of bytes used by key position number in frm */ > +constexpr uint frm_keyno_size= 2; > static inline uint16 read_frm_fieldno(const uchar *data) > { return uint2korr(data); } > -static inline void store_frm_fieldno(const uchar *data, uint16 fieldno) > +static inline void store_frm_fieldno(uchar *data, uint16 fieldno) > +{ int2store(data, fieldno); } > +static inline uint16 read_frm_keyno(const uchar *data) > +{ return uint2korr(data); } > +static inline void store_frm_keyno(uchar *data, uint16 fieldno) > { int2store(data, fieldno); } > +static inline size_t extra2_str_size(size_t len) > +{ return (len > 255 ? 3 : 1) + len; } why did you move that? it's still not used anywhere outside of unireg.cc > > class select_unit; > class TMP_TABLE_PARAM; > diff --git a/sql/unireg.cc b/sql/unireg.cc > index 7130b3e5d8a..ea5d739a97e 100644 > --- a/sql/unireg.cc > +++ b/sql/unireg.cc > @@ -390,7 +385,9 @@ LEX_CUSTRING build_frm_image(THD *thd, const LEX_CSTRING &table, > > if (create_info->period_info.name) > { > - extra2_size+= 1 + extra2_str_size(period_info_len); > + // two extra2 sections are taken after 10.5 This is a confusing comment, it suggests that both extra2 sections were added in 10.5. Remove the comment, please, it's not worth it. > + extra2_size+= 2 + extra2_str_size(period_info_len) > + + extra2_str_size(without_overlaps_len); > } > > bool has_extra2_field_flags_= has_extra2_field_flags(create_fields); > diff --git a/sql/unireg.h b/sql/unireg.h > index 419fbc4bd80..873d6f681fc 100644 > --- a/sql/unireg.h > +++ b/sql/unireg.h > @@ -177,7 +177,8 @@ enum extra2_frm_value_type { > > EXTRA2_ENGINE_TABLEOPTS=128, > EXTRA2_FIELD_FLAGS=129, > - EXTRA2_FIELD_DATA_TYPE_INFO=130 > + EXTRA2_FIELD_DATA_TYPE_INFO=130, > + EXTRA2_PERIOD_WITHOUT_OVERLAPS=131, Please, try to create a table that uses WITHOUT OVERLAPS and open it in 10.4. Just as a test, to make sure it works as expected. > }; > > enum extra2_field_flags { Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 2

[Maria-developers] [MariaDB 10.3.7] Hello, everyone. My process core dumped while calling mysql_ping().
by Ma Allen 14 Jan '20

14 Jan '20

Environment: MariaDB 10.3.7 database server & related headers and libraries CentOS 7.3 or 7.4 ./configure --prefix=/home/spiderflow --localstatedir=/home/spiderflow --enable-unix-socket --with-libnss-libraries=/usr/lib64 --with-libnss-includes=/usr/include/nss3 --with-libnspr-libraries=/usr/lib64 --with-libnspr-includes=/usr/include/nspr4 --enable-netmap --enable-rust --disable-gccmarch-native --enable-mysql --with-libmysql-includes=/usr/local/mariadb-10.3.7-linux-systemd-x86_64/include/mysql --with-libmysql-libraries=/usr/local/mariadb-10.3.7-linux-systemd-x86_64/lib Phenomenon: (gdb) #0 ma_pvio_write (pvio=0x10101010101b2e0, buffer=buffer@entry=0x2510101 "\001", length=length@entry=5) at /home/buildbot/buildbot/build/libmariadb/libmariadb/ma_pvio.c:350 #1 0x00007f7100249dea in ma_net_real_write (net=net@entry=0xee9de0 <g_mysql_info>, packet=0x2510101 "\001", len=5) at /home/buildbot/buildbot/build/libmariadb/libmariadb/ma_net.c:335 #2 0x00007f7100249f60 in ma_net_flush (net=net@entry=0xee9de0 <g_mysql_info>) at /home/buildbot/buildbot/build/libmariadb/libmariadb/ma_net.c:166 #3 0x00007f710024a71a in ma_net_write_command (net=net@entry=0xee9de0 <g_mysql_info>, command=command@entry=14 '\016', packet=packet@entry=0x7f71003d6c67 "", len=0, disable_flush=disable_flush@entry=0 '\000') at /home/buildbot/buildbot/build/libmariadb/libmariadb/ma_net.c:244 #4 0x00007f7100252ca4 in mthd_my_send_cmd (mysql=0xee9de0 <g_mysql_info>, command=<optimized out>, arg=0x7f71003d6c67 "", length=0, skipp_check=<optimized out>, opt_arg=<optimized out>) at /home/buildbot/buildbot/build/libmariadb/libmariadb/mariadb_lib.c:394 #5 0x00007f71002518d0 in mysql_ping (mysql=0xee9de0 <g_mysql_info>) at /home/buildbot/buildbot/build/libmariadb/libmariadb/mariadb_lib.c:2552 #6 0x00000000006651b2 in MySqlWaitConnected (p_mysql_data=p_mysql_data@entry=0xee9de0 <g_mysql_info>) at ../srcSF/spiderFlow-proto-baseline.c:676 #7 0x0000000000668819 in LoadProtoConfInfos (pInfo=pInfo@entry=0xee9de0 <g_mysql_info>) at ../srcSF/spiderFlow-proto-baseline.c:835 #8 0x000000000066c53a in InitBaseLine () at ../srcSF/spiderFlow-proto-baseline.c:1069 #9 0x00000000005f308b in PostConfLoadedSetup (suri=0x10dbb80 <suricata>) at suricata.c:2900 #10 0x0000000000425ce8 in main (argc=5, argv=0x7fff1cdca258) at suricata.c:3072 (gdb) Analyzation: The core dump always exists after running for a while or maybe after the sever restarts. It occurs at the initialization and involves with ONLY ONE thread, which is the main thread of the whole process. The related code is two functions, which both checks the connection and read data from MariaDB. The first function works fine, however, the second one's checking connection leads to core dump. The code to check connection is as follows: void MySqlWaitConnected(MYSQL * p_mysql_data) { while (mysql_ping(p_mysql_data) != 0) { SCLogInfo("Mysql ping error:%s", mysql_error(p_mysql_data)); #ifdef WIN32 Sleep(3000); #else sleep(3); #endif } } Previously, I solved a likely similar problem as follows: Multiple threads shared ONE MySQL handler, which is the MYSQL data structure. Before inserting data into MariaDB without mutex, MySqlWaitConnected() is called. So while multiple threads do the same operation, inserting data is interrupted by mysql_ping() that leads to core dump. Afterwards, I comment MySqlWaitConnected() and add mutex for mysql_real_query() and mysql_commit(). Does anyone meet the same problem before? Any suggestion is appreciated. Thanks in advance. Best Regards, Allen Ma

1 0

Re: [Maria-developers] [Commits] afdd6191d5d: Big Test added for sorting
by Sergey Petrunia 13 Jan '20

13 Jan '20

Hi Varun, As disucssed: - Please fix function names - Please fix the comments to explain what the function is generating - Please don't use session variables. Ok to push after this is addressed On Fri, Jan 03, 2020 at 02:28:25AM +0530, Varun wrote: > revision-id: afdd6191d5dcb004ec9ac0b908871ad8a370da34 (mariadb-10.4.11-18-gafdd6191d5d) > parent(s): 59d4f2a373a7960a533e653877ab69a97e91444a > author: Varun Gupta > committer: Varun Gupta > timestamp: 2020-01-03 02:26:58 +0530 > message: > > Big Test added for sorting > > --- > mysql-test/main/order_by_pack_big.result | 194 +++++++++++++++++++++++++++++++ > mysql-test/main/order_by_pack_big.test | 107 +++++++++++++++++ > 2 files changed, 301 insertions(+) > > diff --git a/mysql-test/main/order_by_pack_big.result b/mysql-test/main/order_by_pack_big.result > new file mode 100644 > index 00000000000..66aad449c38 > --- /dev/null > +++ b/mysql-test/main/order_by_pack_big.result > @@ -0,0 +1,194 @@ > +set @save_rand_seed1= @@RAND_SEED1; > +set @save_rand_seed2= @@RAND_SEED2; > +set @@RAND_SEED1=810763568, @@RAND_SEED2=600681772; > +create table t1(a int); > +insert into t1 select seq from seq_1_to_10000 order by rand(); > +# > +# parameters: > +# mean mean for the column to be considered > +# max_val max_value for the column to be considered > +# > +# This function also calculates the standard deviation > +# which is required to convert standard normal distribution > +# to normal distribution I cannot make any sense of this. The intent of this function is to generate random numbers with the mean of `mean` and standard deviation of ... (max_val - mean) /6 ? > +# > +CREATE FUNCTION f1(mean DOUBLE, max_val DOUBLE) RETURNS DOUBLE > +BEGIN > +DECLARE std_dev DOUBLE DEFAULT 0; > +SET @z= (rand() + rand() + rand() + rand() + rand() + rand() + > +rand() + rand() + rand() + rand() + rand() + rand() - 6); Here we get mean=1, stddev=1. > +SET std_dev= (max_val - mean)/6; > +SET @z= std_dev*@z + mean; ok so we have generated a random number with the mean 'mean' and std_dev as shown above. > +return @z; Please do not use session variables for function-local computations. The variables are in user-session scope, that is, this function will eventually cause a surprise by overwriting user's @z. > +END| > +# > +# parameters: > +# len length of the random string to be generated > +# > +# This function generates a random string for the length passed > +# as an argument with characters in the range of [A,Z] > +# > +CREATE function f2(len INT) RETURNS varchar(128) > +BEGIN > +DECLARE str VARCHAR(256) DEFAULT ''; > +DECLARE x INT DEFAULT 0; > +WHILE (len > 0) DO > +SET x =round(rand()*25); > +SET str= CONCAT(str, CHAR(65 + x)); > +SET len= len-1; > +END WHILE; > +RETURN str; > +END| > +# > +# parameters: > +# mean mean for the column to be considered > +# min_val min_value for the column to be considered > +# max_val max_value for the column to be considered > +# > +CREATE function f3(mean DOUBLE, min_val DOUBLE, max_val DOUBLE) RETURNS INT > +BEGIN > +DECLARE r DOUBLE DEFAULT 0; > +WHILE 1=1 DO > +set r= f1(mean, max_val); > +IF (r >= min_val) THEN > +RETURN round(r); > +end if; > +END WHILE; > +RETURN 0; > +END| > +create table t2 (id INT NOT NULL, a INT, b int); > +insert into t2 select a, f3(12, 0, 64), f3(32, 0, 128) from t1; > +CREATE TABLE t3( > +id INT NOT NULL, > +names VARCHAR(64), > +address VARCHAR(128), > +PRIMARY KEY (id) > +); > +# > +# table t3 stores string calculated from the length stored in > +# table t2 > +# > +insert into t3 select id, f2(a), f2(b) from t2; > +set sort_buffer_size=262144*10; > +flush status; > +select id, > +MD5(group_concat(substring(names,1,3), substring(address,1,3))) > +FROM t3 > +GROUP BY id DIV 100 > +ORDER BY id; > +id MD5(group_concat(substring(names,1,3), substring(address,1,3))) > +10 351239227a41de08388ea422f928cc29 > +149 67299eb34e363edabe31576890087e97 > +232 7ac931ef07a24ebe1293093ec6fa8f3d > +311 8625cade62c8b45c63d8978f8968ebb5 > +430 362761f4180d40372667c8dd7cdcc436 > +502 5380af74db071a35fb1d2491368e641b > +665 d3e3e2a2cb4e0de17c4f12e5b7745802 > +719 5d93632d4c30ec99802f7be7582f4f2d > +883 27747ef400898c7eeeba3ebea8c42fb1 > +942 d1e4ae80ca57b99ee49201b658a7b040 > +1007 fceb25160237c8a3c262735b81d027ac > +1134 cfa9c86c901aaace0e9e94dc6a837468 > +1226 4fb8e9ab9acdd251e7bc51db9e4d2f3b > +1367 e17fa4948562b3411f0b64084de0c605 > +1486 85dd0f507e660600820f106dc8887edf > +1502 5bf6015f936908eed31f5769ad4b0d72 > +1674 01f6c54ea21c4acd26f6c1df6abd793c > +1781 6d38cd061db1f30e2e37cd7d9ac600ad > +1803 2ac17a3853677ffde105735c92a9f2ea > +1969 e1e2e39e9d26baebe23232a429783feb > +2087 af67a443d21665bbb425a783f4e434fa > +2111 1906e379e9ae0b3b580fa134d2a5a146 > +2268 2afaf9091f92fb8e409142552724a85e > +2328 5a8fd5d24c9f7c7bcfbcde84a5b0cfe2 > +2416 d9a69c46523f71fce606c6d6c92ca516 > +2599 55a436a6fb744eefd6878473c34fa41e > +2602 98317430fe15bcc9bb5968b5052c9106 > +2777 8b5c30ae940ff7f31839309b535e3a15 > +2858 0db2f3bcb138c2f91445c4205374a3b4 > +2922 fed051b9185591bc0aaebd1e1471944d > +3027 f0cff102210e7fa32db222ac3444e4cf > +3131 c2f3f5a92d4c2b45cadd9c8cbf04d1be > +3220 8db6dfcca0461654dcb963fe2e1d8f41 > +3331 42031ed42643c755dfd936eb96b28ed5 > +3452 09f418c82012ff6789a6429be0c10f98 > +3519 7d26aac1dbbcff68b528b8c1d80a2c7b > +3680 0ff5b4295168db71b997f6001bba7015 > +3799 3460724c5fc7271a0a3189bf275b9b89 > +3876 13f21a3dfc2bad54c12fffae7cdf8326 > +3937 a240132ca8905b8165bf6e16fa6e7b3a > +4029 5fabf8408215c5bf896eda4e173a8a98 > +4158 c7829b1eeda97ff8c9b2a24ead3f6df6 > +4291 0d24e7e9da38dc44ffb43976560c4730 > +4355 bc804d019300149cb891b8fe8afbe445 > +4461 bb5a658677030b64ca3fd095f8a054fd > +4544 e04f6bfc8dcb8d8014ce39e1b707ed0b > +4646 06af0dd12faee32a07e785c4d56856b8 > +4714 d0c99cc1aead7d06e5323867867d4b00 > +4848 208d1ca5ade34053d92f96937f76380b > +4935 3b62eb6129970e714bdc74565183e183 > +5014 9e19c021b79e32ea6fceb7ced26a3a68 > +5184 41fa16423738302b2fdd6cda8e52f2c9 > +5219 3ab8090c30c0206c1e30ce6cd76cb617 > +5349 bd3e73dd60fbd1819aa468d3d0e6999c > +5400 80dc0e71fcbd2abfec9b585cc04a7545 > +5507 96ed16d40a9e6a1231bc88bd6b3f9c3e > +5672 764347fc7e265a1478c890fa38d8c892 > +5725 6767ae39fec9b789b8b542080162af46 > +5849 41df99caa43ee3f3b162c66c3eb61a44 > +5941 0725e779ca53da50461ef0d3758d819d > +6064 06d28bf28138d5726ab61e51a2e87edc > +6135 b2567b682dd449e358e11c4fb7f7bb72 > +6289 8aa8131d32436add670fed1e7628b297 > +6329 127b1600d2a9f857501f0263536d200b > +6404 266b87348831b9cc5b570e2b16c3006a > +6580 f70b98a00f6adb163c0f89bb6bb6d1ad > +6653 a13a591ba0c88985040c51fda2af7a72 > +6773 ee4306ceb6a3266617707a1ca637c328 > +6822 a8c368cc486b650f6254614535b5b051 > +6938 a7c160cec86018b78942b60b62b5b7fd > +7085 eb360d65bc8080cd5879fb8ddee830cd > +7180 c54bebbb560d9e9196a6f986022d4253 > +7290 4d1820f520483d785ba4e1c89b938f20 > +7390 0d3cd69b8e02fde232df802f3e9fc7a2 > +7449 7328ee3fe9383f891b9af5244c63a0e0 > +7589 467169481de385077ebcad083dd36b0b > +7686 ae22b711e21ba0e0fe20ba713408263a > +7713 e20cd84a1ee8bd1d743947c9c381731d > +7844 bc3f0534e283616d6a4dbb0902c03fa6 > +7935 146ea350d8f1cfef44aa7470cf9e02f8 > +8059 3a88201a77ccbd8ce651eeb555c29fe5 > +8153 9db1e67ef602768b7182401905bacc26 > +8245 c5e6c51763b0bbc1a7e72fe1615f9440 > +8310 ee37ab957141c733350e21a6ed2176f5 > +8432 34ae43ecbfa6c96e12a8c315937d511f > +8596 710f7c0bc4fadbdd859352b584b19d66 > +8647 df6f807e47599027749e1b09b04f6083 > +8742 5efcaddfa993721074a1691947ca611e > +8856 40ad2459d26129770ac6ac2da757ad7e > +8967 344f6b2c8242b9b3bbd09898a80ba4ee > +9057 3084c365110820be5bbfc721f4b2f37d > +9148 13b2a5aa09a1f107f656e848a963e8ea > +9275 908187dba9416102a566b955b29f709e > +9311 d6c8096f5763c6ebdaccb3e2cc3ae686 > +9488 62deb4d1a8900ea7cd7daa1909917490 > +9518 730ecae84924d86922c82152c191d0f6 > +9696 0a15d3446ba3d4b7ca8224633fbab666 > +9752 a74f840a4e599466799d4e0879533da0 > +9887 a7c29b0e5edfcd20572e0fda12a9e9aa > +9903 e89c3ab708646a5d73683ea68c4e366a > +10000 9cc0d2b033602eaea73fa9b2201b01b6 > +show status like '%sort%'; > +Variable_name Value > +Sort_merge_passes 0 > +Sort_priority_queue_sorts 0 > +Sort_range 0 > +Sort_rows 10101 > +Sort_scan 2 > +set sort_buffer_size=default; > +set @@RAND_SEED1= @save_rand_seed1; > +set @@RAND_SEED2= @save_rand_seed2; > +drop function f1; > +drop function f2; > +drop function f3; > +drop table t1, t2, t3; > diff --git a/mysql-test/main/order_by_pack_big.test b/mysql-test/main/order_by_pack_big.test > new file mode 100644 > index 00000000000..021edfee13f > --- /dev/null > +++ b/mysql-test/main/order_by_pack_big.test > @@ -0,0 +1,107 @@ > +--source include/big_test.inc > +--source include/have_sequence.inc > +--source include/have_64bit.inc > + > +set @save_rand_seed1= @@RAND_SEED1; > +set @save_rand_seed2= @@RAND_SEED2; > +set @@RAND_SEED1=810763568, @@RAND_SEED2=600681772; > + > +create table t1(a int); > +insert into t1 select seq from seq_1_to_10000 order by rand(); > +delimiter |; > + > +--echo # > +--echo # parameters: > +--echo # mean mean for the column to be considered > +--echo # max_val max_value for the column to be considered > +--echo # > +--echo # This function also calculates the standard deviation > +--echo # which is required to convert standard normal distribution > +--echo # to normal distribution > +--echo # > + > +CREATE FUNCTION f1(mean DOUBLE, max_val DOUBLE) RETURNS DOUBLE > +BEGIN > + DECLARE std_dev DOUBLE DEFAULT 0; > + SET @z= (rand() + rand() + rand() + rand() + rand() + rand() + > + rand() + rand() + rand() + rand() + rand() + rand() - 6); > + SET std_dev= (max_val - mean)/6; > + SET @z= std_dev*@z + mean; > + return @z; > +END| > + > +--echo # > +--echo # parameters: > +--echo # len length of the random string to be generated > +--echo # > +--echo # This function generates a random string for the length passed > +--echo # as an argument with characters in the range of [A,Z] > +--echo # > + > +CREATE function f2(len INT) RETURNS varchar(128) > +BEGIN > + DECLARE str VARCHAR(256) DEFAULT ''; > + DECLARE x INT DEFAULT 0; > + WHILE (len > 0) DO > + SET x =round(rand()*25); > + SET str= CONCAT(str, CHAR(65 + x)); > + SET len= len-1; > + END WHILE; > +RETURN str; > +END| > + > +--echo # > +--echo # parameters: > +--echo # mean mean for the column to be considered > +--echo # min_val min_value for the column to be considered > +--echo # max_val max_value for the column to be considered > +--echo # > + > +CREATE function f3(mean DOUBLE, min_val DOUBLE, max_val DOUBLE) RETURNS INT > +BEGIN > + DECLARE r DOUBLE DEFAULT 0; > + WHILE 1=1 DO > + set r= f1(mean, max_val); > + IF (r >= min_val) THEN > + RETURN round(r); > + end if; > + END WHILE; > + RETURN 0; > +END| > + > +delimiter ;| > + > +create table t2 (id INT NOT NULL, a INT, b int); > +insert into t2 select a, f3(12, 0, 64), f3(32, 0, 128) from t1; > + > +CREATE TABLE t3( > + id INT NOT NULL, > + names VARCHAR(64), > + address VARCHAR(128), > + PRIMARY KEY (id) > +); > + > +--echo # > +--echo # table t3 stores string calculated from the length stored in > +--echo # table t2 > +--echo # > + > +insert into t3 select id, f2(a), f2(b) from t2; > + > +set sort_buffer_size=262144*10; > +flush status; > +select id, > + MD5(group_concat(substring(names,1,3), substring(address,1,3))) > +FROM t3 > +GROUP BY id DIV 100 > +ORDER BY id; > +show status like '%sort%'; > +set sort_buffer_size=default; > + > +set @@RAND_SEED1= @save_rand_seed1; > +set @@RAND_SEED2= @save_rand_seed2; > + > +drop function f1; > +drop function f2; > +drop function f3; > +drop table t1, t2, t3; > _______________________________________________ > commits mailing list > commits(a)mariadb.org > https://lists.askmonty.org/cgi-bin/mailman/listinfo/commits -- BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

[Maria-developers] MariaDB and reproducible builds: Mroonga, RocksDB and TokuDB
by Otto Kekäläinen 09 Jan '20

09 Jan '20

Reproducible builds are important for software supply chain security. See https://reproducible-builds.org/ All packages in Debian are tested for reproducibility. Currently the latest MariaDB 10.3 build in Debian fails due to the build of the Mroonga, RocksDB and TokuDB plugins. See https://tests.reproducible-builds.org/debian/rb-pkg/unstable/amd64/mariadb-… I've filed issues about this at Mroonga and RocksDB now: https://github.com/mroonga/mroonga/issues/298 https://github.com/facebook/rocksdb/issues/6276 TokuDB does not seem to accept any issues anymore and there are no commits since 2015. The project is dead, right? https://tokutek.atlassian.net/projects/DB/issues https://github.com/percona/tokudb-engine

1 0

Re: [Maria-developers] c91ec05e01b: MDEV-20865 Store foreign key info in TABLE_SHARE
by Sergei Golubchik 07 Jan '20

07 Jan '20

Hi, Aleksey! Just a question for now (and a couple of style comments, I didn't look at the logic yet) On Dec 04, Aleksey Midenkov wrote: > revision-id: c91ec05e01b (mariadb-10.4.4-427-gc91ec05e01b) > parent(s): e6d653b448d > author: Aleksey Midenkov <midenok(a)gmail.com> > committer: Aleksey Midenkov <midenok(a)gmail.com> > timestamp: 2019-11-26 13:04:07 +0300 > message: > > MDEV-20865 Store foreign key info in TABLE_SHARE > > 1. Access foreign keys via TABLE_SHARE::foreign_keys and > TABLE_SHARE::referenced_keys; > > 2. Remove handler FK interface: > > - get_foreign_key_list() > - get_parent_foreign_key_list() > - referenced_by_foreign_key() Good, that was the goal > 3. Invalidate referenced shares on: > > - RENAME TABLE > - DROP TABLE > - RENAME COLUMN > - CREATE TABLE > - ADD FOREIGN KEY > > When foreign table is created or altered by the above operations all > referenced shares are closed. This blocks the operation while any > referenced shares are used (when at least one its TABLE instance is > locked). And this is the main question of this email: Why do you close referenced tables? Minor comments below > diff --git a/sql/handler.h b/sql/handler.h > index e913af1d15d..10984505f70 100644 > --- a/sql/handler.h > +++ b/sql/handler.h > @@ -1030,6 +1031,15 @@ struct TABLE_SHARE; > struct HA_CREATE_INFO; > struct st_foreign_key_info; > typedef struct st_foreign_key_info FOREIGN_KEY_INFO; > +class Table_ident; > +class FK_list : public List<FOREIGN_KEY_INFO> > +{ > +public: > + /* Get all referenced tables for foreign key fk_name. */ > + bool get(THD *thd, std::set<Table_ident> &result, LEX_CSTRING &fk_name, bool foreign); > + /* Get all referenced or foreign tables. */ > + bool get(THD *thd, std::set<Table_ident> &result, bool foreign); Seems unnecessary. Copying a list into a std::set _could_ be justified, if you'd later used it for quick checks "if this table belongs to a set" - something you cannot quickly do with a List. But as far as I can see you copy the List into std::set and then iterate this set. This doesn't make much sense, you can iterate the original list just fine. > +}; > typedef bool (stat_print_fn)(THD *thd, const char *type, size_t type_len, > const char *file, size_t file_len, > const char *status, size_t status_len); > diff --git a/sql/lex_string.h b/sql/lex_string.h > index a62609c6b60..769f4dcbf5e 100644 > --- a/sql/lex_string.h > +++ b/sql/lex_string.h > @@ -41,11 +41,47 @@ class Lex_cstring : public LEX_CSTRING > str= start; > length= end - start; > } > + Lex_cstring(const LEX_CSTRING &src) > + { > + str= src.str; > + length= src.length; > + } > void set(const char *_str, size_t _len) > { > str= _str; > length= _len; > } > + Lex_cstring *strdup_root(MEM_ROOT &mem_root) The way you use it, it looks like you really need a constructor, not a strdup. > + { > + Lex_cstring *dst= > + (Lex_cstring *) alloc_root(&mem_root, sizeof(Lex_cstring)); > + if (!dst) > + return NULL; > + if (!str) > + { > + dst->str= NULL; > + dst->length= 0; > + return dst; > + } > + dst->str= (const char *) memdup_root(&mem_root, str, length + 1); > + if (!dst->str) > + return NULL; > + dst->length= length; > + return dst; > + } > + bool operator< (const Lex_cstring& rhs) const > + { > + return length < rhs.length || (length == rhs.length && memcmp(str, rhs.str, length) < 0); > + } > + bool operator== (const Lex_cstring& rhs) const > + { > + return length == rhs.length && 0 == memcmp(str, rhs.str, length); > + } > + bool operator> (const Lex_cstring& rhs) const > + { > + return length > rhs.length || (length == rhs.length && memcmp(str, rhs.str, length) > 0); > + } Nope. We've been here before, haven't we? Don't overload operators. If you want a method to compare, call it cmp(), or greater_than(), or something btw your comparison has quite weird and unconventional semantics > + > }; Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 15

Re: [Maria-developers] MDEV-21318: Wrong results with window functions and implicit grouping
by Sergey Petrunia 20 Dec '19

20 Dec '19

Hi Varun, > revision-id: fc34657511a9aa08dd92f7363dc53f58934f9673 (mariadb-10.2.29-62-gfc34657511a) > parent(s): f0aa073f2bf3d8d85b3d028df89cdb4cdfc4002d > author: Varun Gupta > committer: Varun Gupta > timestamp: 2019-12-17 16:40:06 +0530 > message: > > MDEV-21318: Wrong results with window functions and implicit grouping > > The issue here is for degenerate joins we should execute the window > function but it is not getting executed in all the cases. > > To get the window function values window function needs to be executed > always. This currently does not happen in few cases > where the join would return 0 or 1 row like > 1) IMPOSSIBLE WHERE > 2) IMPOSSIBLE HAVING > 3) MIN/MAX optimization > 4) EMPTY CONST TABLE > 5) ZERO LIMIT > > The fix is to make sure that window functions get executed > and the temporary table is setup for the execution of window functions > > --- > mysql-test/r/win.result | 51 ++++++++++++++++++++++++++++++++++++++++++++ > mysql-test/t/win.test | 56 +++++++++++++++++++++++++++++++++++++++++++++++++ > sql/sql_select.cc | 33 ++++++++++++++++++++++++++++- > sql/sql_select.h | 1 + > 4 files changed, 140 insertions(+), 1 deletion(-) ... > diff --git a/sql/sql_select.cc b/sql/sql_select.cc > index c9cb533aa33..f6943b18cee 100644 > --- a/sql/sql_select.cc > +++ b/sql/sql_select.cc > @@ -1447,6 +1447,7 @@ JOIN::optimize_inner() > zero_result_cause= "Zero limit"; > } > table_count= top_join_tab_count= 0; > + implicit_grouping_with_window_funcs(); not needed + redundant (I'll elaborate below) > error= 0; > goto setup_subq_exit; > } > @@ -1502,6 +1503,7 @@ JOIN::optimize_inner() > zero_result_cause= "No matching min/max row"; > table_count= top_join_tab_count= 0; > error=0; > + implicit_grouping_with_window_funcs(); not needed + redundant (I'll elaborate below) > goto setup_subq_exit; > } > if (res > 1) > @@ -1517,6 +1519,7 @@ JOIN::optimize_inner() > tables_list= 0; // All tables resolved > select_lex->min_max_opt_list.empty(); > const_tables= top_join_tab_count= table_count; > + implicit_grouping_with_window_funcs(); This doesn't seem to be needed? > /* > Extract all table-independent conditions and replace the WHERE > clause with them. All other conditions were computed by opt_sum_query > @@ -1615,6 +1618,7 @@ JOIN::optimize_inner() > zero_result_cause= "no matching row in const table"; > DBUG_PRINT("error",("Error: %s", zero_result_cause)); > error= 0; > + implicit_grouping_with_window_funcs(); OK > goto setup_subq_exit; > } > if (!(thd->variables.option_bits & OPTION_BIG_SELECTS) && > @@ -1639,6 +1643,7 @@ JOIN::optimize_inner() > zero_result_cause= > "Impossible WHERE noticed after reading const tables"; > select_lex->mark_const_derived(zero_result_cause); > + implicit_grouping_with_window_funcs(); Ok > goto setup_subq_exit; > } > > @@ -1781,6 +1786,7 @@ JOIN::optimize_inner() > zero_result_cause= > "Impossible WHERE noticed after reading const tables"; > select_lex->mark_const_derived(zero_result_cause); > + implicit_grouping_with_window_funcs(); > goto setup_subq_exit; There is no test coverage for this case. Please add. > } > > @@ -18225,7 +18231,8 @@ void set_postjoin_aggr_write_func(JOIN_TAB *tab) > } > } > else if (join->sort_and_group && !tmp_tbl->precomputed_group_by && > - !join->sort_and_group_aggr_tab && join->tables_list) > + !join->sort_and_group_aggr_tab && join->tables_list && > + join->top_join_tab_count) > { > DBUG_PRINT("info",("Using end_write_group")); > aggr->set_write_func(end_write_group); > @@ -26924,6 +26931,30 @@ Item *remove_pushed_top_conjuncts(THD *thd, Item *cond) > return cond; > } > > +/* > + There are 5 cases in which we shortcut the join optimization process as we > + conclude that the join would be a degenerate one > + 1) IMPOSSIBLE WHERE > + 2) IMPOSSIBLE HAVING > + 3) MIN/MAX optimization (@see opt_sum_query) > + 4) EMPTY CONST TABLE > + 5) ZERO LIMIT > + If a window function is present in any of the above cases then to get the > + result of the window function, we need to execute it. So we need to Do we really need to do this for cases #2 and #5? In these case, no ouptut is produced, why compute the window functions? > + create a temporary table for its execution. Here we need to take in mind > + that aggregate functions and non-aggregate function need not be executed. > + > +*/ > + > + > +void JOIN::implicit_grouping_with_window_funcs() The name of the function doesn't look right. It's a noun ("subject") while most functions start with verb (e.g. JOIN::optimize_...) handle_implicit_grouping_with_window_funcs() isn't ideal either but would be better I think. > +{ > + if (select_lex->have_window_funcs() && send_row_on_empty_set()) > + { > + const_tables= top_join_tab_count= table_count= 0; > + } > +} > + > /** > @} (end of group Query_Optimizer) > */ BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

Re: [Maria-developers] [MariaDB/server] Proper locking for mysql.gtid_slave_pos truncation (84b437d)
by Kristian Nielsen 18 Dec '19

18 Dec '19

Sergey Vojtovich <notifications(a)github.com> writes: > ATTN @dr-m, @andrelkin, @SachinSetiya, @knielsen So IIUC, this is about incorrect usage of ha_truncate() in rpl_slave_state::truncate_state_table(). This is used only for SET GLOBAL gtid_slave_pos = "..." when all slave threads are stopped and nothing else is accessing the gtid_pos table. So it's fine to use ha_truncate() if that can be done easily (and correctly). But it would also be fine just to loop and delete all rows one by one in a normal transaction, if that is simpler. gtid_slave_pos is a small table, there are normally only a few rows per active replication domain. I'm not myself very familiar with details of metadata locking etc. around ha_truncate(). But looking at the code now, I don't understand why it only truncates one table? If --gtid-pos-auto-engines is in effect, there could be multiple tables... shouldn't they all be cleared when setting the gtid_slave_pos variable? If so, maybe the delete-rows-one-by-one approach is in any case preferable over ha_truncate, since it can then be done transactionally, to not leave an inconsistent gtid_slave_pos state if one truncate fails and another succeeds? Hope this helps, - Kristian.

2 2

Re: [Maria-developers] f72427f463d: MDEV-20923:UBSAN: member access within address Б─╕ which does not point to an object of type 'xid_count_per_binlog'
by andrei.elkin＠pp.inet.fi 17 Dec '19

17 Dec '19

Sujatha, Kristian, howdy. Sorry, I managed to sent my view without actually cc-ing Kristian, as claimed. X-From-Line: nobody Wed Nov 27 19:27:51 2019 From: Andrei Elkin <andrei.elkin(a)mariadb.com> To: sujatha <sujatha.sivakumar(a)mariadb.com> Cc: <commits(a)mariadb.org>, <maria-developers(a)lists.launchpad.net> Subject: Re: f72427f463d: MDEV-20923:UBSAN: member access within address Let me try that again now. Kristian, the question was about `xid_count_per_binlog' struct comment -- .. struct xid_count_per_binlog : public ilink { char *binlog_name; uint binlog_name_len; ulong binlog_id; /* Total prepared XIDs and pending checkpoint requests in this binlog. */ long xid_count; /* For linking in requests to the binlog background thread. */ xid_count_per_binlog *next_in_queue; xid_count_per_binlog(); /* Give link error if constructor used. */ .. -----------------------------^ }; Now I am guessing a constructor was attempted to fail out.. Was that related to de-POD-ing of the object ('cos of the constructor)? It's pretty obscure. We would appreciate your explanation. The former review mail is quoted (except one more [@]-marked view note, Sujatha) further. Cheers, Andrei ------------------------------------------------------------------------------- Sujatha, hello. The patch looks quite good. I have two suggestions though. The first is to match the destructor's free() with actual allocation (of what is to be freed) in the new parameterized constructor. And 2nd-ly, I suggest to leave the explicit zero argument constructor intact. Kristian is cc-d to help us to decide. My comments are below. Cheers, Andrei sujatha <sujatha.sivakumar(a)mariadb.com> writes: > revision-id: f72427f463d316a54ebf87c2e84c73947e3c5fe4 (mariadb-10.1.43-5-gf72427f463d) > parent(s): 13db50fc03e7312e6c01b06c7e4af69f69ba5382 > author: Sujatha > committer: Sujatha > timestamp: 2019-11-12 16:11:16 +0530 > message: > > MDEV-20923:UBSAN: member access within address … which does not point to an object of type 'xid_count_per_binlog' > > Problem: > ------- > Accessing a member within 'xid_count_per_binlog' structure results in > following error when 'UBSAN' is enabled. > > member access within address 0xXXX which does not point to an object of type > 'xid_count_per_binlog' > > Analysis: > --------- > The problem appears to be that no constructor for 'xid_count_per_binlog' is > being called, and thus the vtable will not be initialized. > > Fix: > --- > Defined a parameterized constructor for 'xid_count_per_binlog' class. > > --- > sql/log.cc | 28 ++++++++++++++-------------- > sql/log.h | 9 ++++++++- > 2 files changed, 22 insertions(+), 15 deletions(-) > > diff --git a/sql/log.cc b/sql/log.cc > index acf1f8f8a9c..2b8b67febef 100644 > --- a/sql/log.cc > +++ b/sql/log.cc > @@ -3216,7 +3216,7 @@ void MYSQL_BIN_LOG::cleanup() > DBUG_ASSERT(!binlog_xid_count_list.head()); > WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::cleanup(): Removing xid_list_entry " > "for %s (%lu)", b); > - my_free(b); > + delete b; > } > > mysql_mutex_destroy(&LOCK_log); > @@ -3580,17 +3580,17 @@ bool MYSQL_BIN_LOG::open(const char *log_name, > */ > uint off= dirname_length(log_file_name); > uint len= strlen(log_file_name) - off; > - char *entry_mem, *name_mem; > - if (!(new_xid_list_entry = (xid_count_per_binlog *) > - my_multi_malloc(MYF(MY_WME), > - &entry_mem, sizeof(xid_count_per_binlog), > - &name_mem, len, > - NULL))) > + char *name_mem; > + name_mem= (char *) my_malloc(len, MYF(MY_ZEROFILL)); > + if (!name_mem) > goto err; > memcpy(name_mem, log_file_name+off, len); That is both my_malloc and memcpy to go into the new constructor to match [*] (find the label below). > - new_xid_list_entry->binlog_name= name_mem; > - new_xid_list_entry->binlog_name_len= len; > - new_xid_list_entry->xid_count= 0; > + new_xid_list_entry= new xid_count_per_binlog(name_mem, (int)len); > + if (!new_xid_list_entry) > + { > + my_free(name_mem); > + goto err; > + } > > /* > Find the name for the Initial binlog checkpoint. > @@ -3711,7 +3711,7 @@ bool MYSQL_BIN_LOG::open(const char *log_name, > { > WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::open(): Removing xid_list_entry for " > "%s (%lu)", b); > - my_free(binlog_xid_count_list.get()); > + delete binlog_xid_count_list.get(); > } > mysql_cond_broadcast(&COND_xid_list); > WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::open(): Adding new xid_list_entry for " > @@ -3758,7 +3758,7 @@ Turning logging off for the whole duration of the MySQL server process. \ > To turn it on again: fix the cause, \ > shutdown the MySQL server and restart it.", name, errno); > if (new_xid_list_entry) > - my_free(new_xid_list_entry); > + delete new_xid_list_entry; [@] With `delete` we don't need the non-null check: `if()` to remove. > if (file >= 0) > mysql_file_close(file, MYF(0)); > close(LOG_CLOSE_INDEX); > @@ -4252,7 +4252,7 @@ bool MYSQL_BIN_LOG::reset_logs(THD *thd, bool create_new_log, > DBUG_ASSERT(b->xid_count == 0); > WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::reset_logs(): Removing " > "xid_list_entry for %s (%lu)", b); > - my_free(binlog_xid_count_list.get()); > + delete binlog_xid_count_list.get(); > } > mysql_cond_broadcast(&COND_xid_list); > reset_master_pending--; > @@ -9736,7 +9736,7 @@ TC_LOG_BINLOG::mark_xid_done(ulong binlog_id, bool write_checkpoint) > break; > WSREP_XID_LIST_ENTRY("TC_LOG_BINLOG::mark_xid_done(): Removing " > "xid_list_entry for %s (%lu)", b); > - my_free(binlog_xid_count_list.get()); > + delete binlog_xid_count_list.get(); > } > > mysql_mutex_unlock(&LOCK_xid_list); > diff --git a/sql/log.h b/sql/log.h > index b4c9b24a3a9..30a55e577a4 100644 > --- a/sql/log.h > +++ b/sql/log.h > @@ -587,7 +587,14 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG > long xid_count; > /* For linking in requests to the binlog background thread. */ > xid_count_per_binlog *next_in_queue; > - xid_count_per_binlog(); /* Give link error if constructor used. */ Maybe we should leave it as it seems to be for catching inadvertent object constructions. I'm cc-ing Kristian to clear out. > + xid_count_per_binlog(char *log_file_name, uint log_file_name_len) > + :binlog_name(log_file_name), binlog_name_len(log_file_name_len), > + binlog_id(0), xid_count(0) > + {} > + ~xid_count_per_binlog() > + { > + my_free(binlog_name); [*] > + } > }; > I_List<xid_count_per_binlog> binlog_xid_count_list; > mysql_mutex_t LOCK_binlog_background_thread; > _______________________________________________ > commits mailing list > commits(a)mariadb.org > https://lists.askmonty.org/cgi-bin/mailman/listinfo/commits

2 1

Re: [Maria-developers] 6ac19d09c66: MDEV-16978 Application-time periods: WITHOUT OVERLAPS
by Sergei Golubchik 17 Dec '19

17 Dec '19

Hi, Nikita! On Nov 15, Nikita Malyavin wrote: > revision-id: 6ac19d09c66 (mariadb-10.4.4-280-g6ac19d09c66) > parent(s): 67ddb6507d5 > author: Nikita Malyavin <nikitamalyavin(a)gmail.com> > committer: Nikita Malyavin <nikitamalyavin(a)gmail.com> > timestamp: 2019-08-20 17:56:07 +1000 > message: > > MDEV-16978 Application-time periods: WITHOUT OVERLAPS > > * The overlaps check is implemented on a handler level per row > command. It creates a separate cursor (actually, another handler > instance) and caches it inside the original handler, when > ha_update_row or ha_insert_row is issued. Cursor closes on unlocking > the handler. > > * Containing the same key in index means unique constraint violation > even in usual terms. So we fetch left and right neighbours and check > that they have same key prefix, excluding from the key only the period > part. If it doesnt match, then there's no such neighbour, and the > check passes. Otherwise, we check if this neighbour intersects with > the considered key. > > * the check does introduce new error and fails with ER_DUPP_KEY error. > This might break REPLACE workflow and should be fixed separately > > diff --git a/sql/handler.cc b/sql/handler.cc > index 94cffd69b75..560f6316602 100644 > --- a/sql/handler.cc > +++ b/sql/handler.cc > @@ -6913,6 +6928,130 @@ void handler::set_lock_type(enum thr_lock_type lock) > table->reginfo.lock_type= lock; > } > > +int handler::ha_check_overlaps(const uchar *old_data, const uchar* new_data) > +{ > + DBUG_ASSERT(new_data); > + if (!table_share->period.unique_keys) > + return 0; > + if (table->versioned() && !table->vers_end_field()->is_max()) > + return 0; > + > + bool is_update= old_data != NULL; > + if (!check_overlaps_buffer) > + check_overlaps_buffer= (uchar*)alloc_root(&table_share->mem_root, > + table_share->max_unique_length > + + table_share->reclength); > + auto *record_buffer= check_overlaps_buffer + table_share->max_unique_length; > + auto *handler= this; > + if (handler->inited != NONE) > + { > + if (!check_overlaps_handler) > + { > + check_overlaps_handler= clone(table_share->normalized_path.str, > + &table_share->mem_root); wouldn't it be simpler and cheaper to use HA_EXTRA_REMEMBER_POS and HA_EXTRA_RESTORE_POS, as we've discussed? > + int error= -1; > + if (check_overlaps_handler != NULL) > + error= check_overlaps_handler->ha_external_lock(table->in_use, F_RDLCK); > + if (error) > + return error; > + } > + handler= check_overlaps_handler; > + } why "if (handler->inited != NONE)" ? What happens if it is NONE? > + > + for (uint key_nr= 0; key_nr < table_share->keys; key_nr++) > + { > + const KEY *key_info= table->key_info + key_nr; > + const uint key_parts= key_info->user_defined_key_parts; > + if (!key_info->without_overlaps) > + continue; > + > + key_copy(check_overlaps_buffer, new_data, key_info, 0); > + if (is_update) > + { > + bool key_used= false; > + for (uint k= 0; k < key_parts && !key_used; k++) > + key_used= bitmap_is_set(table->write_set, > + key_info->key_part[k].fieldnr - 1); > + if (!key_used) > + continue; > + } Good. > + > + int error= handler->ha_index_init(key_nr, 0); > + if (error) > + return error; > + > + for (int run= 0; run < 2; run++) > + { > + if (run == 0) > + { > + error = handler->ha_index_read_map(record_buffer, > + check_overlaps_buffer, > + key_part_map((1 << key_parts) - 1), > + HA_READ_KEY_OR_PREV); > + if (error == HA_ERR_KEY_NOT_FOUND) > + continue; > + } > + else > + { > + error = handler->ha_index_next(record_buffer); > + if (error == HA_ERR_END_OF_FILE) > + continue; > + } > + > + if (error) > + { > + handler->ha_index_end(); > + return error; > + } I found this "for (int run= 0; run < 2; run++)" rather confusing. Particularly as you have an if() to have different code for the first and the second runs. it would be clearer to unroll the loop and move the common code (period comparison) into a function. Like handler->ha_index_read_map(...); compare_periods(); handler->ha_index_next(); compare_periods(); But it can be done better. You search for the key with the same value and a period start <= than the period start of new row. And then you have to check two rows for overlaps. If you'll search for a key with the period start <= than the _period end_ of the new row, then you'll only need to check one row (may be one more, if updating). Also, why do you store both period ends in the index? It seems like it'd be enough to store only one end - only the start or only the end. Both ends help if you use keyreads, but you don't. On the second thought, perhaps, you should use keyreads, there's no need to fetch the whole row for this overlap check. On the yet another thought it might not work with HA_EXTRA_REMEMBER_POS. > + > + /* In case of update it could appear that the nearest neighbour is > + * a record we are updating. It means, that there are no overlaps > + * from this side. */ > + if (is_update && memcmp(old_data + table->s->null_bytes, > + record_buffer + table->s->null_bytes, > + table->s->reclength - table->s->null_bytes) == 0) > + { > + continue; > + } I'd rather compare row positions here. > + > + uint period_key_part_nr= key_parts - 2; > + int cmp_res= 0; > + for (uint part_nr= 0; !cmp_res && part_nr < period_key_part_nr; part_nr++) > + { > + Field *f= key_info->key_part[part_nr].field; > + cmp_res= f->cmp(f->ptr_in_record(new_data), > + f->ptr_in_record(record_buffer)); > + } > + if (cmp_res) > + continue; /* key is different => no overlaps */ > + > + int period_cmp[2][2]= {/* l1 > l2, l1 > r2, r1 > l2, r1 > r2 */}; > + for (int i= 0; i < 2; i++) > + { > + for (int j= 0; j < 2; j++) > + { > + Field *lhs= key_info->key_part[period_key_part_nr + i].field; > + Field *rhs= key_info->key_part[period_key_part_nr + j].field; > + > + period_cmp[i][j]= lhs->cmp(lhs->ptr_in_record(new_data), > + rhs->ptr_in_record(record_buffer)); > + } > + } this can be simplified too if you'll change the code as I suggested above to do only one index read - you'll only need one comparison, instead of four. > + > + if ((period_cmp[0][0] <= 0 && period_cmp[1][0] > 0) > + || (period_cmp[0][0] >= 0 && period_cmp[0][1] < 0)) > + { > + handler->ha_index_end(); > + return HA_ERR_FOUND_DUPP_KEY; > + } > + } > + error= handler->ha_index_end(); > + if (error) > + return error; > + } > + return 0; > +} > + > #ifdef WITH_WSREP > /** > @details > diff --git a/sql/handler.h b/sql/handler.h > index 63d0bf2215c..71debf9bab7 100644 > --- a/sql/handler.h > +++ b/sql/handler.h > @@ -3237,6 +3243,8 @@ class handler :public Sql_alloc > DBUG_ASSERT(cached_table_flags < (HA_LAST_TABLE_FLAG << 1)); > return cached_table_flags; > } > + /** PRIMARY KEY WITHOUT OVERLAPS check is done globally */ what do you mean "globally"? > + int ha_check_overlaps(const uchar *old_data, const uchar* new_data); > /** > These functions represent the public interface to *users* of the > handler class, hence they are *not* virtual. For the inheritance > diff --git a/sql/sql_table.cc b/sql/sql_table.cc > index 9bb1d98152b..5aba86003c6 100644 > --- a/sql/sql_table.cc > +++ b/sql/sql_table.cc > @@ -4549,25 +4552,57 @@ static bool vers_prepare_keys(THD *thd, HA_CREATE_INFO *create_info, > if (key->type != Key::PRIMARY && key->type != Key::UNIQUE) > continue; > > - Key_part_spec *key_part= NULL; > - List_iterator<Key_part_spec> part_it(key->columns); > - while ((key_part=part_it++)) > + if (create_info->versioned()) > { > - if (!my_strcasecmp(system_charset_info, > - row_start_field, > - key_part->field_name.str) || > - > - !my_strcasecmp(system_charset_info, > - row_end_field, > - key_part->field_name.str)) > - break; > + Key_part_spec *key_part=NULL; > + List_iterator<Key_part_spec> part_it(key->columns); > + while ((key_part=part_it++)) > + { > + if (row_start_field.streq(key_part->field_name) || > + row_end_field.streq(key_part->field_name)) > + break; > + } > + if (!key_part) > + key->columns.push_back(new Key_part_spec(&row_end_field, 0)); > } > - if (key_part) > - continue; // Key already contains Sys_start or Sys_end > + } > > - Key_part_spec *key_part_sys_end_col= > - new (thd->mem_root) Key_part_spec(&create_info->vers_info.as_row.end, 0); > - key->columns.push_back(key_part_sys_end_col); > + key_it.rewind(); > + while ((key=key_it++)) > + { > + if (key->without_overlaps) > + { > + if (key->type != Key::PRIMARY && key->type != Key::UNIQUE) > + { > + my_error(ER_PERIOD_WITHOUT_OVERLAPS_NON_UNIQUE, MYF(0), key->period.str); > + return true; > + } > + if (!create_info->period_info.is_set() > + || !key->period.streq(create_info->period_info.name)) > + { > + my_error(ER_PERIOD_NOT_FOUND, MYF(0), key->period.str); > + return true; > + } > + if (thd->work_part_info) > + { > + my_error(ER_PERIOD_WITHOUT_OVERLAPS_PARTITIONED, MYF(0)); why? > + return true; > + } > + const auto &period_start= create_info->period_info.period.start; > + const auto &period_end= create_info->period_info.period.end; > + List_iterator<Key_part_spec> part_it(key->columns); > + while (Key_part_spec *key_part= part_it++) > + { > + if (period_start.streq(key_part->field_name) > + || period_end.streq(key_part->field_name)) > + { > + my_error(ER_KEY_CONTAINS_PERIOD_FIELDS, MYF(0), key->name.str); > + return true; > + } > + } > + key->columns.push_back(new Key_part_spec(&period_start, 0)); > + key->columns.push_back(new Key_part_spec(&period_end, 0)); > + } > } > > return false; > diff --git a/sql/table.h b/sql/table.h > index 2b866159fe0..6511960488e 100644 > --- a/sql/table.h > +++ b/sql/table.h > @@ -1730,10 +1731,18 @@ class IS_table_read_plan; > > /** number of bytes used by field positional indexes in frm */ > constexpr uint frm_fieldno_size= 2; > +/** number of bytes used by key position number in frm */ > +constexpr uint frm_keyno_size= 2; > static inline uint16 read_frm_fieldno(const uchar *data) > { return uint2korr(data); } > -static inline void store_frm_fieldno(const uchar *data, uint16 fieldno) > +static inline void store_frm_fieldno(uchar *data, uint16 fieldno) > { int2store(data, fieldno); } > +static inline uint16 read_frm_keyno(const uchar *data) > +{ return uint2korr(data); } > +static inline void store_frm_keyno(uchar *data, uint16 fieldno) > +{ int2store(data, fieldno); } > +static inline size_t frm_ident_stored_size(size_t len) > +{ return len + (len > 255 ? 3 : 1); } this is exactly extra2_str_size() function. Don't duplicate it, reuse. (and rename, if you'd like a more generic name) > > class select_unit; > class TMP_TABLE_PARAM; > diff --git a/sql/unireg.cc b/sql/unireg.cc > index 7130b3e5d8a..1e95242786e 100644 > --- a/sql/unireg.cc > +++ b/sql/unireg.cc > @@ -485,6 +486,16 @@ LEX_CUSTRING build_frm_image(THD *thd, const LEX_CSTRING &table, > store_frm_fieldno(pos, get_fieldno_by_name(create_info, create_fields, > create_info->period_info.period.end)); > pos+= frm_fieldno_size; > + store_frm_keyno(pos, create_info->period_info.unique_keys); > + pos+= frm_keyno_size; > + for (uint key= 0; key < keys; key++) > + { > + if (key_info[key].without_overlaps) > + { > + store_frm_keyno(pos, key); > + pos+= frm_keyno_size; > + } > + } This will make 10.5 frms look corrupted in 10.4. Better use a new EXTRA2 flag for that. With a number > EXTRA2_ENGINE_IMPORTANT. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 20

[Maria-developers] MariaDB for CentOS 8
by Thomas Stephen Lee 12 Dec '19

12 Dec '19

Hi, Please release MariaDB 10.4 for CentOS 8. https://downloads.mariadb.org/mariadb/repositories/ does not show CentOS 8. thanks -- Thomas Stephen Lee

3 5

Re: [Maria-developers] 387839e44cc: Plugin for Hashicorp Vault Key Management System
by Sergei Golubchik 04 Dec '19

04 Dec '19

Hi, Julius! Just a couple of comments, see below: On Dec 04, Julius Goryavsky wrote: > revision-id: 387839e44cc (mariadb-10.4.10-179-g387839e44cc) > parent(s): 8115a02283d > author: Julius Goryavsky <julius.goryavsky(a)mariadb.com> > committer: Julius Goryavsky <julius.goryavsky(a)mariadb.com> > timestamp: 2019-11-15 13:37:35 +0100 > message: > > Plugin for Hashicorp Vault Key Management System > diff --git a/plugin/hashicorp_key_management/hashicorp_key_management_plugin.cc b/plugin/hashicorp_key_management/hashicorp_key_management_plugin.cc > new file mode 100644 > index 00000000000..32c0e6417a3 > --- /dev/null > +++ b/plugin/hashicorp_key_management/hashicorp_key_management_plugin.cc ... > + if (tolen_len > max_token_size) > + { > + my_printf_error(ER_UNKNOWN_ERROR, > + "Maximum allowed token length exceeded", > + 0); > + return true; > + } > + size_t buf_len = x_vault_token_len + tolen_len + 1; typo: "token_len" I think > +#ifdef _WIN32 > + char *token_header = (char *) _alloca(sizeof(char) * buf_len); > +#else > + char *token_header = (char *) alloca(sizeof(char) * buf_len); > +#endif Please, remove all these ifdefs and add one at the very top of the file: #ifdef _WIN32 #define alloca _alloca #endif Here I presume that you've tested it in buildbot and alloca is indeed present on all builders and we don't need a fallback here. Also remove sizeof(char) everywhere, it's always 1. C99 standard says 6.5.3.4 The sizeof operator ... When applied to an operand that has type char, unsigned char, or signed char, (or a qualified version thereof) the result is 1. > + snprintf(token_header, buf_len, "%s%s", x_vault_token, token); > + curl_errbuf[0] = '\0'; > + if ((list= curl_slist_append(list, token_header)) == NULL || > + (curl_res= curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errbuf)) != > + CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, > + write_response_memory)) != CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_WRITEDATA, > + &read_data_stream)) != > + CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list)) != > + CURLE_OK || ... > + /* > + If the result of the conversion does not fit into an > + unsigned integer or if an error (for example, overflow) > + occurred during the conversion, then we need to return > + an error: > + */ > +#if ULONG_MAX > UINT_MAX > + if (version > UINT_MAX || errno) > +#else > + if (errno) > +#endif did you verify that otherwise it makes compilation to fail? better to keep the number of ifdefs in the code to a minimum, so don't add this one unless absolutely necessary > + { > + return ENCRYPTION_KEY_VERSION_INVALID; > + } > + return (unsigned int) version; > +} Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] [Commits] 3a11b49bb5b: MDEV-20900: IN predicate to IN subquery conversion causes performance regression
by Sergey Petrunia 03 Dec '19

03 Dec '19

Hi Varun, As far as I understand, the intent of the patch is that this check: + for (uint i=0; i < n; i++) + { + if (item1->element_index(i)->cmp_type() != + item2->element_index(i)->cmp_type()) matches this check in subquery_types_allow_materialization() if (!inner->type_handler()->subquery_type_allows_materialization(inner, outer)) which has implementations like so: bool Type_handler_real_result:: subquery_type_allows_materialization(const Item *inner, const Item *outer) const { DBUG_ASSERT(inner->cmp_type() == REAL_RESULT); return outer->cmp_type() == REAL_RESULT; } // .. and the same for other datatypes ... I would like to see comments in both places (cmp_row_types and subquery_types_allow_materialization()) explaining that. Note that some Type_handler::subquery_types_allow_materialization() have a more complex implementation, for example Type_handler_string_result::subquery_types_allow_materialization() compares collations. Is there any reasons why cmp_row_types() doesn't use the same call as subquery_types_allow_materialization does: if (!inner->type_handler()->subquery_type_allows_materialization(inner, outer)) ? On Mon, Nov 04, 2019 at 04:38:02PM +0530, Varun wrote: > revision-id: 3a11b49bb5b716538f98c6a212bbbfa6fc9b7a88 (mariadb-10.3.17-145-g3a11b49bb5b) > parent(s): 162f475c4be81dfbceed093ad03d114b4c69a3c0 > author: Varun Gupta > committer: Varun Gupta > timestamp: 2019-11-04 16:28:25 +0530 > message: > > MDEV-20900: IN predicate to IN subquery conversion causes performance regression > > Disable the IN predicate to IN subquery conversion when the types on the left and > right hand side of the IN predicate are not of comparable type. > > --- > mysql-test/main/opt_tvc.result | 53 ++++++++++++++++++++++++++++++++++++++---- > mysql-test/main/opt_tvc.test | 31 ++++++++++++++++++++++++ > sql/sql_tvc.cc | 27 ++++++++++++++++++++- > 3 files changed, 106 insertions(+), 5 deletions(-) > > diff --git a/mysql-test/main/opt_tvc.result b/mysql-test/main/opt_tvc.result > index 5329a9f64be..a68e70e8a25 100644 > --- a/mysql-test/main/opt_tvc.result > +++ b/mysql-test/main/opt_tvc.result > @@ -629,11 +629,9 @@ SELECT * FROM t1 WHERE i IN (NULL, NULL, NULL, NULL, NULL); > i > EXPLAIN EXTENDED SELECT * FROM t1 WHERE i IN (NULL, NULL, NULL, NULL, NULL); > id select_type table type possible_keys key key_len ref rows filtered Extra > -1 PRIMARY t1 ALL NULL NULL NULL NULL 3 100.00 > -1 PRIMARY <derived3> ALL NULL NULL NULL NULL 5 100.00 Using where; FirstMatch(t1); Using join buffer (flat, BNL join) > -3 DERIVED NULL NULL NULL NULL NULL NULL NULL NULL No tables used > +1 SIMPLE t1 ALL NULL NULL NULL NULL 3 100.00 Using where > Warnings: > -Note 1003 /* select#1 */ select `test`.`t1`.`i` AS `i` from `test`.`t1` semi join ((values (NULL),(NULL),(NULL),(NULL),(NULL)) `tvc_0`) where `test`.`t1`.`i` = `tvc_0`.`_col_1` > +Note 1003 select `test`.`t1`.`i` AS `i` from `test`.`t1` where `test`.`t1`.`i` in (NULL,NULL,NULL,NULL,NULL) > SET in_predicate_conversion_threshold= default; > DROP TABLE t1; > # > @@ -687,3 +685,50 @@ f1 f2 > 1 1 > DROP TABLE t1,t2,t3; > SET @@in_predicate_conversion_threshold= default; > +# > +# MDEV-20900: IN predicate to IN subquery conversion causes performance regression > +# > +create table t1(a int, b int); > +insert into t1 select seq-1, seq-1 from seq_1_to_10; > +set in_predicate_conversion_threshold=2; > +explain select * from t1 where t1.a IN ("1","2","3","4"); > +id select_type table type possible_keys key key_len ref rows Extra > +1 SIMPLE t1 ALL NULL NULL NULL NULL 10 Using where > +select * from t1 where t1.a IN ("1","2","3","4"); > +a b > +1 1 > +2 2 > +3 3 > +4 4 > +set in_predicate_conversion_threshold=0; > +explain select * from t1 where t1.a IN ("1","2","3","4"); > +id select_type table type possible_keys key key_len ref rows Extra > +1 SIMPLE t1 ALL NULL NULL NULL NULL 10 Using where > +select * from t1 where t1.a IN ("1","2","3","4"); > +a b > +1 1 > +2 2 > +3 3 > +4 4 > +set in_predicate_conversion_threshold=2; > +explain select * from t1 where (t1.a,t1.b) in (("1","1"),(2,2),(3,3),(4,4)); > +id select_type table type possible_keys key key_len ref rows Extra > +1 SIMPLE t1 ALL NULL NULL NULL NULL 10 Using where > +select * from t1 where (t1.a,t1.b) in (("1","1"),(2,2),(3,3),(4,4)); > +a b > +1 1 > +2 2 > +3 3 > +4 4 > +set in_predicate_conversion_threshold=0; > +explain select * from t1 where (t1.a,t1.b) in (("1","1"),(2,2),(3,3),(4,4)); > +id select_type table type possible_keys key key_len ref rows Extra > +1 SIMPLE t1 ALL NULL NULL NULL NULL 10 Using where > +select * from t1 where (t1.a,t1.b) in (("1","1"),(2,2),(3,3),(4,4)); > +a b > +1 1 > +2 2 > +3 3 > +4 4 > +drop table t1; > +SET @@in_predicate_conversion_threshold= default; > diff --git a/mysql-test/main/opt_tvc.test b/mysql-test/main/opt_tvc.test > index 7319dbdc9e8..e4e8c6d7919 100644 > --- a/mysql-test/main/opt_tvc.test > +++ b/mysql-test/main/opt_tvc.test > @@ -3,6 +3,7 @@ > # > source include/have_debug.inc; > source include/default_optimizer_switch.inc; > +source include/have_sequence.inc; > > create table t1 (a int, b int); > > @@ -397,3 +398,33 @@ SELECT * FROM t3 WHERE (f1,f2) IN ((2, 2), (1, 2), (3, 5), (1, 1)); > DROP TABLE t1,t2,t3; > > SET @@in_predicate_conversion_threshold= default; > + > +--echo # > +--echo # MDEV-20900: IN predicate to IN subquery conversion causes performance regression > +--echo # > + > +create table t1(a int, b int); > +insert into t1 select seq-1, seq-1 from seq_1_to_10; > + > +set in_predicate_conversion_threshold=2; > + > +let $query= select * from t1 where t1.a IN ("1","2","3","4"); > +eval explain $query; > +eval $query; > + > +set in_predicate_conversion_threshold=0; > +eval explain $query; > +eval $query; > + > +set in_predicate_conversion_threshold=2; > +let $query= select * from t1 where (t1.a,t1.b) in (("1","1"),(2,2),(3,3),(4,4)); > +eval explain $query; > +eval $query; > + > +set in_predicate_conversion_threshold=0; > +eval explain $query; > +eval $query; > + > +drop table t1; > +SET @@in_predicate_conversion_threshold= default; > + > diff --git a/sql/sql_tvc.cc b/sql/sql_tvc.cc > index 816c6fe1089..78c7c34a81a 100644 > --- a/sql/sql_tvc.cc > +++ b/sql/sql_tvc.cc > @@ -796,6 +796,31 @@ bool Item_subselect::wrap_tvc_into_select(THD *thd, st_select_lex *tvc_sl) > } > > > +/* > + @brief > + Check whether the items are of comparable type or not > + > + @retval > + 0 comparable > + 1 not comparable > +*/ > + > +static bool cmp_row_types(Item* item1, Item* item2) > +{ > + uint n= item1->cols(); > + if (item2->check_cols(n)) > + return true; > + > + for (uint i=0; i < n; i++) > + { > + if (item1->element_index(i)->cmp_type() != > + item2->element_index(i)->cmp_type()) > + return true; > + } > + return false; > +} > + > + > /** > @brief > Transform IN predicate into IN subquery > @@ -843,7 +868,7 @@ Item *Item_func_in::in_predicate_to_in_subs_transformer(THD *thd, > > for (uint i=1; i < arg_count; i++) > { > - if (!args[i]->const_item()) > + if (!args[i]->const_item() || cmp_row_types(args[0], args[i])) > return this; > } > > _______________________________________________ > commits mailing list > commits(a)mariadb.org > https://lists.askmonty.org/cgi-bin/mailman/listinfo/commits -- BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

Re: [Maria-developers] 387839e44cc: Plugin for Hashicorp Vault Key Management System
by Sergei Golubchik 02 Dec '19

02 Dec '19

Hi, Julius! Looks good! A couple of minor comments, see below On Dec 02, Julius Goryavsky wrote: > revision-id: 387839e44cc (mariadb-10.4.10-179-g387839e44cc) > parent(s): 8115a02283d > author: Julius Goryavsky <julius.goryavsky(a)mariadb.com> > committer: Julius Goryavsky <julius.goryavsky(a)mariadb.com> > timestamp: 2019-11-15 13:37:35 +0100 > message: > > Plugin for Hashicorp Vault Key Management System > > diff --git a/plugin/hashicorp_key_management/CMakeLists.txt b/plugin/hashicorp_key_management/CMakeLists.txt > new file mode 100644 > index 00000000000..4b5f79afa66 > --- /dev/null > +++ b/plugin/hashicorp_key_management/CMakeLists.txt > @@ -0,0 +1,11 @@ > +INCLUDE(FindCURL) > +IF(NOT CURL_FOUND) > + # Can't build plugin > + RETURN() > +ENDIF() > + > +INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql ${CURL_INCLUDE_DIR}) > + > +MYSQL_ADD_PLUGIN(HASHICORP_KEY_MANAGEMENT > + hashicorp_key_management_plugin.cc > + LINK_LIBRARIES ${CURL_LIBRARIES}) > diff --git a/plugin/hashicorp_key_management/hashicorp_key_management_plugin.cc b/plugin/hashicorp_key_management/hashicorp_key_management_plugin.cc > new file mode 100644 > index 00000000000..26ee7b7cd3d > --- /dev/null > +++ b/plugin/hashicorp_key_management/hashicorp_key_management_plugin.cc > @@ -0,0 +1,325 @@ > +/* Copyright (C) 2019 MariaDB Corporation > + > + This program is free software; you can redistribute it and/or modify > + it under the terms of the GNU General Public License as published by > + the Free Software Foundation; version 2 of the License. > + > + This program is distributed in the hope that it will be useful, > + but WITHOUT ANY WARRANTY; without even the implied warranty of > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > + GNU General Public License for more details. > + > + You should have received a copy of the GNU General Public License > + along with this program; if not, write to the Free Software > + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ > + > +#include <mysql/plugin_encryption.h> > +#include <mysqld_error.h> > +#include <string.h> > +#include <stdlib.h> > +#include <limits.h> > +#include <errno.h> > +#include <string> > +#include <sstream> > +#include <curl/curl.h> > + > +static char* vault_url; > +static char* token; > +static char* vault_ca; > + > +static MYSQL_SYSVAR_STR(vault_ca, vault_ca, > + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, > + "Path to Certificate Authority (CA) bundle (is a file " > + "that contains root and intermediate certificates)", > + NULL, NULL, ""); > + > +static MYSQL_SYSVAR_STR(vault_url, vault_url, > + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, > + "HTTP[s] URL that is used to connect to the Hashicorp Vault server", > + NULL, NULL, "https://127.0.0.1:8200/v1/secret"); > + > +static MYSQL_SYSVAR_STR(token, token, > + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY | PLUGIN_VAR_NOSYSVAR, > + "Authentication token that passed to the Hashicorp Vault " > + "in the request header", > + NULL, NULL, ""); > + > +static struct st_mysql_sys_var* settings[] = { > + MYSQL_SYSVAR(vault_url), > + MYSQL_SYSVAR(token), > + MYSQL_SYSVAR(vault_ca), > + NULL > +}; > + > +static std::string get_error_from_curl (CURLcode curl_code, char *curl_errbuf) > +{ > + size_t len = strlen(curl_errbuf); > + std::ostringstream stream; > + if (curl_code != CURLE_OK) > + { > + stream << "CURL returned this error code: " << curl_code; > + stream << " with error message : "; > + if (len) > + stream << curl_errbuf; > + else > + stream << curl_easy_strerror(curl_code); > + } > + return stream.str(); > +} > + > +#define max_response_size 65536 > + > +static size_t write_response_memory (void *contents, size_t size, size_t nmemb, > + void *userp) > +{ > + size_t realsize = size * nmemb; > + if (size != 0 && realsize / size != nmemb) > + return 0; // overflow detected > + std::ostringstream *read_data = static_cast<std::ostringstream*>(userp); > + size_t current_length = read_data->tellp(); > + if (current_length + realsize > max_response_size) > + return 0; // response size limit exceeded > + read_data->write(static_cast<char*>(contents), realsize); > + if (!read_data->good()) > + return 0; > + return realsize; > +} > + > +#define timeout 300 > + > +static bool curl_run (char *url, std::string *response) > +{ > + char curl_errbuf [CURL_ERROR_SIZE]; > + struct curl_slist *list = NULL; > + std::ostringstream read_data_stream; > + CURLcode curl_res = CURLE_OK; > + long http_code = 0; > + CURL *curl = curl_easy_init(); > + if (curl == NULL) > + { > + my_printf_error(ER_UNKNOWN_ERROR, > + "Cannot initialize curl session", > + ME_ERROR_LOG_ONLY); > + return true; > + } > + std::string token_header = std::string("X-Vault-Token:") + > + std::string(token); > + curl_errbuf[0] = '\0'; > + if ((list= curl_slist_append(list, token_header.c_str())) == NULL || > + (curl_res= curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errbuf)) != > + CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, > + write_response_memory)) != CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_WRITEDATA, > + &read_data_stream)) != > + CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list)) != > + CURLE_OK || > + /* > + The options CURLOPT_SSL_VERIFYPEER and CURLOPT_SSL_VERIFYHOST are > + set explicitly to withstand possible future changes in curl defaults: > + */ > + (curl_res= curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1)) != > + CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L)) != > + CURLE_OK || > + (vault_ca != NULL && strlen(vault_ca) != 0 && vault_ca still cannot be NULL :) > + (curl_res= curl_easy_setopt(curl, CURLOPT_CAINFO, vault_ca)) != > + CURLE_OK) || > + (curl_res= curl_easy_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_ALL)) != > + CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L)) != > + CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, timeout)) != > + CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout)) != > + CURLE_OK) > + { if you get an error here you won't free list, so it'll be a memory leak. In fact, I don't understand why you need two if() here, just combine them in one, the error message is the same anyway, and the second if() already has curl_slist_free_all(). > + my_printf_error(ER_UNKNOWN_ERROR, > + get_error_from_curl(curl_res, curl_errbuf).c_str(), > + ME_ERROR_LOG_ONLY); > + return true; > + } > + if ((curl_res = curl_easy_setopt(curl, CURLOPT_URL, url)) != CURLE_OK || > + (curl_res = curl_easy_perform(curl)) != CURLE_OK || > + (curl_res = curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, > + &http_code)) != CURLE_OK) > + { > + curl_slist_free_all(list); > + my_printf_error(ER_UNKNOWN_ERROR, > + get_error_from_curl(curl_res, curl_errbuf).c_str(), > + ME_ERROR_LOG_ONLY); > + return true; > + } > + curl_slist_free_all(list); > + if (http_code == 404) > + { > + *response = std::string(""); > + return false; > + } > + *response = read_data_stream.str(); > + return http_code < 200 || http_code >= 300; > +} > + > +static unsigned int get_latest_version (uint key_id) > +{ > + char url[2048]; > + std::string response_str; > + const char *response; > + const char *js, *ver; > + int js_len, ver_len; > + size_t response_len; > + snprintf(url, sizeof(url), "%s/%u", vault_url, key_id); > + if (curl_run(url, &response_str)) { > + my_printf_error(ER_UNKNOWN_ERROR, > + "Unable to get key data", > + 0); > + return ENCRYPTION_KEY_VERSION_INVALID; > + } > + response = response_str.c_str(); > + response_len = response_str.size(); > + if (json_get_object_key(response, response + response_len, > + "metadata", &js, &js_len) != JSV_OBJECT) > + { > + return ENCRYPTION_KEY_VERSION_INVALID; > + } > + if (json_get_object_key(js, js+js_len,"version", > + &ver, &ver_len) != JSV_NUMBER) > + { > + return ENCRYPTION_KEY_VERSION_INVALID; > + } > + /* > + Here we may use atoi, but it has undefined overflow behavior > + and does not work with unsigned integers, so use strtoul: > + */ > + unsigned long version = strtoul(ver, NULL, 10); > + /* > + If the result of the conversion does not fit into an > + unsigned integer or if an error (for example, overflow) > + occurred during the conversion, then we need to return > + an error: > + */ > + if (version > UINT_MAX || errno) this might be an error with gcc, "condition is always true" double check this please, no need to fix anything if there's no error > + { > + return ENCRYPTION_KEY_VERSION_INVALID; > + } > + return version; this might be an error with VS, because of the implicit ulong->uint cast. double check this please, no need to fix anything if there's no error > +} > + > +static unsigned int get_key_from_vault (unsigned int key_id, > + unsigned int key_version, > + unsigned char *dstbuf, > + unsigned int *buflen) > +{ > + char url[2048]; > + std::string response_str; > + const char *response; > + const char *js, *ver, *key; > + int js_len, key_len, ver_len; > + size_t response_len; > + if (key_version != ENCRYPTION_KEY_VERSION_INVALID) > + snprintf(url, sizeof(url), "%s/%u?%u", vault_url, key_id, key_version); > + else > + snprintf(url, sizeof(url), "%s/%u", vault_url, key_id); > + if (curl_run(url, &response_str)) > + { > + my_printf_error(ER_UNKNOWN_ERROR, > + "Unable to get key data", > + 0); > + return ENCRYPTION_KEY_VERSION_INVALID; > + } > + response = response_str.c_str(); > + response_len = response_str.size(); > +#ifndef NDEBUG > + /* > + An internal check that is needed only for debugging the plugin > + operation - in order to ensure that we get from the Hashicorp Vault > + server exactly the version of the key that is needed: > + */ > + if (key_version != ENCRYPTION_KEY_VERSION_INVALID) > + { > + if (json_get_object_key(response, response + response_len, > + "metadata", &js, &js_len) != JSV_OBJECT) > + { > + return ENCRYPTION_KEY_VERSION_INVALID; > + } > + if (json_get_object_key(js, js+js_len,"version", > + &ver, &ver_len) != JSV_NUMBER) > + { > + return ENCRYPTION_KEY_VERSION_INVALID; > + } > + /* > + Here we may use atoi, but it has undefined overflow behavior > + and does not work with unsigned integers, so use strtoul: > + */ > + unsigned long version = strtoul(ver, NULL, 10); > + /* > + If the result of the conversion does not fit into an > + unsigned integer or if an error (for example, overflow) > + occurred during the conversion, then we need to return > + an error: > + */ > + if (errno || key_version != version) > + { > + return ENCRYPTION_KEY_VERSION_INVALID; > + } > + } > +#endif > + if (json_get_object_key(response, response + response_len, > + "data", &js, &js_len) != JSV_OBJECT) > + { > + return ENCRYPTION_KEY_VERSION_INVALID; > + } > + if (json_get_object_key(js, js + js_len, "data", > + &key, &key_len) != JSV_STRING) > + { > + return ENCRYPTION_KEY_VERSION_INVALID; > + } > + if (*buflen < (unsigned) key_len) { > + *buflen= key_len; > + return ENCRYPTION_KEY_BUFFER_TOO_SMALL; > + } > + *buflen= key_len; > + memcpy(dstbuf, key, key_len); > + return 0; > +} > + > +struct st_mariadb_encryption hashicorp_key_management_plugin= { > + MariaDB_ENCRYPTION_INTERFACE_VERSION, > + get_latest_version, > + get_key_from_vault, > + 0, 0, 0, 0, 0 > +}; > + > +static int hashicorp_key_management_plugin_init(void *p) > +{ > + curl_global_init(CURL_GLOBAL_ALL); > + return 0; > +} > + > +static int hashicorp_key_management_plugin_deinit(void *p) > +{ > + curl_global_cleanup(); > + return 0; > +} > + > +/* > + Plugin library descriptor > +*/ > +maria_declare_plugin(hashicorp_key_management) > +{ > + MariaDB_ENCRYPTION_PLUGIN, > + &hashicorp_key_management_plugin, > + "hashicorp_key_management", > + "MariaDB Corporation", > + "HashiCorp Vault key management plugin", > + PLUGIN_LICENSE_GPL, > + hashicorp_key_management_plugin_init, > + hashicorp_key_management_plugin_deinit, > + 0x0100 /* 1.0 */, > + NULL, /* status variables */ > + settings, > + "1.0", > + MariaDB_PLUGIN_MATURITY_ALPHA > +} > +maria_declare_plugin_end; Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 387839e44cc: Plugin for Hashicorp Vault Key Management System
by Sergei Golubchik 28 Nov '19

28 Nov '19

Hi, Julius! On Nov 28, Julius Goryavsky wrote: > Hi, Sergey, > > many thanks for the detailed review, I fixed all the flaws and left only > what raises questions or where I have explanations: > > >> + if ((list= curl_slist_append(list, token_header.c_str())) == NULL || > >> + (list= curl_slist_append(list, "Content-Type: application/json")) > == NULL || > > > > Why do you specify Content-Type header if you aren't sending any content? > > Seems redundant. > > I have already come across servers that respond with the wrong content-type > if the input request had a different content-type or if the input request > does not contain a content-type at all. Fortunately, the Hashicorp server > works without setting the correct content-type in the input, but may be > more reliable to set it explicitly? I don't understand. Content-Type in the request header does not mean that you *accept* application/json, it means you are *sending* application/json. And you are not, you are sending no body at all, just the header. > >> + (vault_ca != NULL && strlen(vault_ca) != 0 && (curl_res= > curl_easy_setopt(curl, CURLOPT_CAINFO , vault_ca)) != CURLE_OK) || > > > How can vault_ca == NULL here? > > I think it cannot, the check is redundant too. > > I don’t know, we have a guarantee that if this option is not set, then the > corresponding variable will always point to an empty zero-terminated > string, but will never just be NULL instead? (This is the path to the file > with the CA bundle. In principle, a Hashicorp Vault can work without it, it > is needed only if there is no root or intermediate certificates on the > other side.) Yes, I believe vault_ca can never be NULL. The default value is "", so it'll be either that or a string specified on the command line. > > > + std::string ver_str = std::string(ver, ver_len); > > > + return strtoul(ver_str.c_str(), NULL, 10); > > > Why do you need to do that? create std::string, copy/reallocate it? > > What's the point? Just do atol(ver) > > Does json_get_object_key return a new zero-terminated string? atol cannot > be applied to a fragment with a length (without trailing zero), as far as I > know. No, it doesn't return a zero-terminated string. It returns a pointer into a valid json string. So you can be sure that ver[ver_len] is a non-digit character (it'll be '\n' or, may be, '}'). Which is enough for atol(). > >> + return ENCRYPTION_KEY_VERSION_INVALID; > >> + return decode_data(key, key_len, dstbuf, buflen); > > > Why? I don't see that in hashicorp docs that the key is base64-encoded > > But after all, as far as I understand at this level, the key for us is > arbitrary binary data? What if it contains all kinds of quotation marks, > commas, or, UTF-8 escape codes and all kinds of unprintable characters like > a zero? I thought that it should be base64-endoded if it is binary data, so > that we can pass it through JSON encoding in the form of text that will not > cause a failure in the JSON parser. I believe that with the proper escaping json can handle any binary data just fine. With your base64 requirement you basically mean that one cannot use 3rd party tools to create/manage keys, unless the tool has an option to store the keys base64-encoded. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 387839e44cc: Plugin for Hashicorp Vault Key Management System
by Sergei Golubchik 27 Nov '19

27 Nov '19

Hi, Julius! On Nov 27, Julius Goryavsky wrote: > revision-id: 387839e44cc (mariadb-10.4.10-179-g387839e44cc) > parent(s): 8115a02283d > author: Julius Goryavsky <julius.goryavsky(a)mariadb.com> > committer: Julius Goryavsky <julius.goryavsky(a)mariadb.com> > timestamp: 2019-11-15 13:37:35 +0100 > message: > > Plugin for Hashicorp Vault Key Management System > > diff --git a/plugin/hashicorp_key_management/hashicorp_key_management_plugin.cc b/plugin/hashicorp_key_management/hashicorp_key_management_plugin.cc > new file mode 100644 > index 00000000000..6de1d9d1f3a > --- /dev/null > +++ b/plugin/hashicorp_key_management/hashicorp_key_management_plugin.cc > @@ -0,0 +1,330 @@ > +/* Copyright (C) 2019 MariaDB Corporation > + > + This program is free software; you can redistribute it and/or modify > + it under the terms of the GNU General Public License as published by > + the Free Software Foundation; version 2 of the License. > + > + This program is distributed in the hope that it will be useful, > + but WITHOUT ANY WARRANTY; without even the implied warranty of > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > + GNU General Public License for more details. > + > + You should have received a copy of the GNU General Public License > + along with this program; if not, write to the Free Software > + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ > + > +#include <mysql/plugin.h> This is redundant, because plugin_encryption.h includes plugin.h But it does not harm either, so if you want to include it explictly for clarity - sure, do it > +#include <mysql/plugin_encryption.h> > +#include <mysqld_error.h> > +#include <string.h> > +#include <stdlib.h> > +#include <string> > +#include <sstream> > +#include <curl/curl.h> > + > +static char* vault_url; > +static char* token; > +static char* vault_ca; > + > +static MYSQL_SYSVAR_STR(vault_ca, vault_ca, > + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, > + "Path to Certificate Authority (CA) bundle (is a file " > + "that contains root and intermediate certificates)", > + NULL, NULL, ""); > + > +static MYSQL_SYSVAR_STR(vault_url, vault_url, > + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, > + "HTTP[s] URL that is used to connect to the Hashicorp Vault server", > + NULL, NULL, "https://127.0.0.1:8200/v1/secret"); > + > +static MYSQL_SYSVAR_STR(token, token, > + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY | PLUGIN_VAR_NOSYSVAR, > + "Authentication token that passed to the Hashicorp Vault " > + "in the request header", > + NULL, NULL, ""); > + > +static struct st_mysql_sys_var* settings[] = { > + MYSQL_SYSVAR(vault_url), > + MYSQL_SYSVAR(token), > + MYSQL_SYSVAR(vault_ca), > + NULL > +}; > + > +static std::string get_error_from_curl (CURLcode curl_code, char *curl_errbuf) > +{ > + size_t len = strlen(curl_errbuf); > + std::ostringstream stream; > + if (curl_code != CURLE_OK) > + { > + stream << "CURL returned this error code: " << curl_code; > + stream << " with error message : "; > + if (len) > + stream << curl_errbuf; > + else > + stream << curl_easy_strerror(curl_code); > + } > + return stream.str(); > +} > + > +#define max_response_size 65536 > + > +static size_t write_response_memory (void *contents, size_t size, size_t nmemb, void *userp) > +{ > + size_t realsize = size * nmemb; > + if (size != 0 && realsize / size != nmemb) > + return 0; // overflow > + std::ostringstream *read_data = static_cast<std::ostringstream*>(userp); > + size_t ss_pos = read_data->tellp(); > + read_data->seekp(0, std::ios::end); > + size_t number_of_read_bytes = read_data->tellp(); > + read_data->seekp(ss_pos); > + if (number_of_read_bytes + realsize > max_response_size) > + return 0; // response size limit exceeded > + read_data->write(static_cast<char*>(contents), realsize); > + if (!read_data->good()) > + return 0; > + return realsize; > +} > + > +#define timeout 300 > + > +static bool setup_curl_session (CURL *curl, char *curl_errbuf, > + curl_slist **list_ptr, > + std::ostringstream &read_data_stream) > +{ > + struct curl_slist *list = NULL; > + std::string token_header = std::string("X-Vault-Token:") + > + std::string(token); > + CURLcode curl_res = CURLE_OK; > + read_data_stream.str(""); > + read_data_stream.clear(); Seems redundant, the stream was just created > + curl_errbuf[0] = '\0'; > + if ((list= curl_slist_append(list, token_header.c_str())) == NULL || > + (list= curl_slist_append(list, "Content-Type: application/json")) == NULL || Why do you specify Content-Type header if you aren't sending any content? Seems redundant. > + (curl_res= curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errbuf)) != CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_response_memory)) != CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_WRITEDATA, static_cast<void *>(&read_data_stream))) != CURLE_OK || Do you need to cast here? Seems redundant. > + (curl_res= curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list)) != CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1)) != CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L)) != CURLE_OK || This is redundant, CURLOPT_SSL_VERIFYPEER is 1 by default, CURLOPT_SSL_VERIFYHOST is 2 by default. But again, if you want to be explicit for the sake of clarity - feel free to > + (vault_ca != NULL && strlen(vault_ca) != 0 && (curl_res= curl_easy_setopt(curl, CURLOPT_CAINFO, vault_ca)) != CURLE_OK) || How can vault_ca == NULL here? I think it cannot, the check is redundant too. > + (curl_res= curl_easy_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_ALL)) != CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L)) != CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, timeout)) != CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout)) != CURLE_OK || > + (curl_res = curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, (long)CURL_HTTP_VERSION_1_1)) != CURLE_OK) Why? > + { > + *list_ptr = list; > + my_printf_error(ER_UNKNOWN_ERROR, > + get_error_from_curl(curl_res, curl_errbuf).c_str(), > + ME_ERROR_LOG_ONLY); > + return true; > + } > + *list_ptr = list; > + return false; > +} > + > +static bool curl_run (char * url, const char * * response) > +{ > + char curl_errbuf [CURL_ERROR_SIZE]; > + struct curl_slist *list = NULL; > + std::ostringstream read_data_stream; > + CURLcode curl_res = CURLE_OK; > + long http_code = 0; > + CURL *curl = curl_easy_init(); > + if (curl == NULL) > + { > + my_printf_error(ER_UNKNOWN_ERROR, > + "Cannot initialize curl session", > + ME_ERROR_LOG_ONLY); > + return true; > + } > + if (setup_curl_session(curl, curl_errbuf, &list, read_data_stream) || your setup_curl_session doesn't make much sense now. It could've been useful if one could prepare the session once and then reuse it. but the way you do it now, it's just draws a rather arbitrary line where some one of the curl setup happens inside setup_curl_session function and the rest in the caller. better remove setup_curl_session and put that big if() here. > + (curl_res = curl_easy_setopt(curl, CURLOPT_URL, url)) != CURLE_OK || > + (curl_res = curl_easy_perform(curl)) != CURLE_OK || > + (curl_res = curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, > + &http_code)) != CURLE_OK) > + { > + if (list != NULL) > + { > + curl_slist_free_all(list); > + } > + my_printf_error(ER_UNKNOWN_ERROR, > + get_error_from_curl(curl_res, curl_errbuf).c_str(), > + ME_ERROR_LOG_ONLY); > + return true; > + } > + if (list != NULL) > + { > + curl_slist_free_all(list); > + } > + if (http_code == 404) > + { > + *response = NULL; > + return false; > + } > + *response = read_data_stream.str().c_str(); Really? Who owns this string? Will read_data_stream free it automatically when it gets destructed? > + return http_code < 200 || http_code >= 300; > +} > + > +static unsigned int get_latest_version (uint key_id) > +{ > + char url[2048]; > + const char *response; > + const char *js, *ver; > + int js_len, ver_len; > + size_t response_len; > + snprintf(url, sizeof(url), "%s/%u", vault_url, key_id); > + if (curl_run(url, &response)) { > + my_printf_error(ER_UNKNOWN_ERROR, "Unable to get key data", 0); > + return ENCRYPTION_KEY_VERSION_INVALID; > + } > + response_len = strlen(response); > + if (json_get_object_key(response, response + response_len, > + "metadata", &js, &js_len) != JSV_OBJECT) > + return ENCRYPTION_KEY_VERSION_INVALID; > + if (json_get_object_key(js, js+js_len,"version", > + &ver, &ver_len) != JSV_NUMBER) > + { > + return ENCRYPTION_KEY_VERSION_INVALID; > + } > + std::string ver_str = std::string(ver, ver_len); > + return strtoul(ver_str.c_str(), NULL, 10); Why do you need to do that? create std::string, copy/reallocate it? What's the point? Just do atol(ver) > +} > + > +static int decode_data (const char *src, int src_len, > + unsigned char* dstbuf, unsigned *buflen) > +{ > + int length_of_memory_needed_for_decode = > + my_base64_needed_decoded_length(src_len); > + char* data= (char *)malloc( > + length_of_memory_needed_for_decode * sizeof(char)); > + int decoded_length= > + my_base64_decode(src, src_len, data, NULL, 0); > + if (decoded_length <= 0) > + { > + memset(data, 0, length_of_memory_needed_for_decode); > + free(data); > + return ENCRYPTION_KEY_BUFFER_TOO_SMALL; > + } > + if (*buflen < (unsigned) decoded_length) { > + *buflen= decoded_length; > + free(data); > + return ENCRYPTION_KEY_BUFFER_TOO_SMALL; > + } > + *buflen= decoded_length; > + memcpy(dstbuf, data, decoded_length); > + free(data); > + return 0; > +} > + > +static unsigned int get_key_from_vault (unsigned int key_id, > + unsigned int key_version, > + unsigned char* dstbuf, > + unsigned int *buflen) > +{ > + char url[2048]; > + const char *response; > + const char *js, *ver, *key; > + int js_len, key_len, ver_len; > + size_t response_len; > + if (key_version != ENCRYPTION_KEY_VERSION_INVALID) > + snprintf(url, sizeof(url), "%s/%u?%u", vault_url, key_id, key_version); > + else > + snprintf(url, sizeof(url), "%s/%u", vault_url, key_id); > + if (curl_run(url, &response)) > + { > + my_printf_error(ER_UNKNOWN_ERROR, "Unable to get key data", 0); > + return ENCRYPTION_KEY_VERSION_INVALID; > + } > + response_len = strlen(response); > + if (key_version != ENCRYPTION_KEY_VERSION_INVALID) > + { > + if (json_get_object_key(response, response + response_len, > + "metadata", &js, &js_len) != JSV_OBJECT) > + { > + return ENCRYPTION_KEY_VERSION_INVALID; > + } > + if (json_get_object_key(js, js+js_len,"version", > + &ver, &ver_len) != JSV_NUMBER) > + { > + return ENCRYPTION_KEY_VERSION_INVALID; > + } > +#ifndef NDEBUG > + /* > + An internal check that is needed only for debugging the plugin > + operation - in order to ensure that we get from the Hashicorp Vault > + server exactly the version of the key that is needed: > + */ > + std::string ver_str = std::string(ver, ver_len); same redundant std::string > + if (strtoul(ver_str.c_str(), NULL, 10) != key_version) > + return ENCRYPTION_KEY_VERSION_INVALID; > +#endif > + } > + if (json_get_object_key(response, response + response_len, > + "data", &js, &js_len) != JSV_OBJECT) > + return ENCRYPTION_KEY_VERSION_INVALID; > + if (json_get_object_key(js, js + js_len, "data", &key, &key_len) != JSV_STRING) > + return ENCRYPTION_KEY_VERSION_INVALID; > + return decode_data(key, key_len, dstbuf, buflen); Why? I don't see that in hashicorp docs that the key is base64-encoded > +} > + > +struct st_mariadb_encryption hashicorp_key_management_plugin= { > + MariaDB_ENCRYPTION_INTERFACE_VERSION, > + get_latest_version, > + get_key_from_vault, > + 0, 0, 0, 0, 0 > +}; > + > +static int hashicorp_key_management_plugin_init(void *p) > +{ > + curl_global_init(CURL_GLOBAL_ALL); > + return 0; > +} > + > +static int hashicorp_key_management_plugin_deinit(void *p) > +{ > + curl_global_cleanup(); > + return 0; > +} > + > +/* > + Plugin library descriptor > +*/ > +maria_declare_plugin(hashicorp_key_management) > +{ > + MariaDB_ENCRYPTION_PLUGIN, > + &hashicorp_key_management_plugin, > + "hashicorp_key_management", > + "MariaDB Corporation", > + "HashiCorp Vault key management plugin", > + PLUGIN_LICENSE_GPL, > + hashicorp_key_management_plugin_init, > + hashicorp_key_management_plugin_deinit, > + 0x0100 /* 1.0 */, > + NULL, /* status variables */ > + settings, > + "1.0", > + MariaDB_PLUGIN_MATURITY_ALPHA > +} > +maria_declare_plugin_end; Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] c23c5671a60: Simplified version
by Sergei Golubchik 23 Nov '19

23 Nov '19

Hi, Julius! First, please combine your two commits in one. There's no historical value in keeping the first overcomplicated implementation in the tree. See other comments below: On Nov 23, Julius Goryavsky wrote: > revision-id: c23c5671a60 (mariadb-10.4.10-180-gc23c5671a60) > parent(s): 387839e44cc > author: Julius Goryavsky <julius.goryavsky(a)mariadb.com> > committer: Julius Goryavsky <julius.goryavsky(a)mariadb.com> > timestamp: 2019-11-22 18:38:39 +0100 > message: > > Simplified version > > diff --git a/plugin/hashicorp_key_management/CMakeLists.txt b/plugin/hashicorp_key_management/CMakeLists.txt > new file mode 100644 > index 00000000000..95ff2434976 > --- /dev/null > +++ b/plugin/hashicorp_key_management/CMakeLists.txt > @@ -0,0 +1,20 @@ > +SET(HASHICORP_KEY_MANAGEMENT_PLUGIN_SOURCES > + hashicorp_key_management_plugin.cc) > + > +IF (WITH_CURL) > + INCLUDE(FindCURL) > + IF(CURL_FOUND) > + ELSE() > + # Can't build plugin > + RETURN() > + ENDIF() > +ELSE() > + # Can't build plugin > + RETURN() > +ENDIF() Better just INCLUDE(FindCURL) IF(NOT CURL_FOUND) RETURN() ENDIF() No need to check for -DWITH_CURL=ON, it should be enough to specify -DHASHICORP_KEY_MANAGEMENT_PLUGIN=ON if one wants a plugin, the plugin should implicitly look for curl. > + > +INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql ${CURL_INCLUDE_DIR}) > + > +MYSQL_ADD_PLUGIN(HASHICORP_KEY_MANAGEMENT > + ${HASHICORP_KEY_MANAGEMENT_PLUGIN_SOURCES} I wouldn't introduce HASHICORP_KEY_MANAGEMENT_PLUGIN_SOURCES variable just for one file, but do as you like > + LINK_LIBRARIES ${CURL_LIBRARIES}) > diff --git a/plugin/hashicorp_key_management/hashicorp_key_management_plugin.cc b/plugin/hashicorp_key_management/hashicorp_key_management_plugin.cc > new file mode 100644 > index 00000000000..3dc5214323b > --- /dev/null > +++ b/plugin/hashicorp_key_management/hashicorp_key_management_plugin.cc > @@ -0,0 +1,405 @@ > +/* Copyright (C) 2019 MariaDB Corporation > + > + This program is free software; you can redistribute it and/or modify > + it under the terms of the GNU General Public License as published by > + the Free Software Foundation; version 2 of the License. > + > + This program is distributed in the hope that it will be useful, > + but WITHOUT ANY WARRANTY; without even the implied warranty of > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > + GNU General Public License for more details. > + > + You should have received a copy of the GNU General Public License > + along with this program; if not, write to the Free Software > + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ > + > +#include <my_global.h> > +#include <typelib.h> > +#include "sql_error.h" these three headers shouldn't be included, only use plugin services, do not include server headers at random. > +#include <mysqld_error.h> > +#include <mysql/plugin_encryption.h> > +#include <string.h> > +#include <string> > +#include <sstream> > +#include <curl/curl.h> > + > +static char* vault_url; > +static char* secret_mount_point; > +static char* token; > +static char* vault_ca; > + > +static unsigned long encryption_algorithm; > + > +static const char *encryption_algorithm_names[]= > +{ > + "aes_cbc", > +#ifdef HAVE_EncryptAes128Ctr > + "aes_ctr", > +#endif > + 0 > +}; This is what file plugin does, I don't think you need to copy it here. Just use the default algorithm. > + > +static TYPELIB encryption_algorithm_typelib= > +{ > + array_elements(encryption_algorithm_names)-1,"", > + encryption_algorithm_names, NULL > +}; > + > +static MYSQL_SYSVAR_STR(vault_ca, vault_ca, > + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, > + "Path to Certificate Authority (CA) bundle", > + NULL, NULL, ""); > + > +static MYSQL_SYSVAR_STR(vault_url, vault_url, > + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, > + "Hashicorp Vault URL", > + NULL, NULL, ""); > + > +static MYSQL_SYSVAR_STR(secret_mount_point, secret_mount_point, > + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, > + "Secret mount point", > + NULL, NULL, ""); I don't see a point in splitting the url in two parts that are _always_ used together. Please, remove the "secret_mount_point" variable, the user will specify it as a part of the url. > + > +static MYSQL_SYSVAR_STR(token, token, > + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, > + "Authentication token", > + NULL, NULL, ""); Here, perhaps, I'd specify PLUGIN_VAR_NOSYSVAR flag. To hide the token from SHOW VARIABLES, because the token is all one needs to retrieve encryption keys. > + > +#ifdef HAVE_EncryptAes128Ctr > +#define recommendation ", aes_ctr is the recommended one" > +#else > +#define recommendation "" > +#endif > +static MYSQL_SYSVAR_ENUM(encryption_algorithm, encryption_algorithm, > + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, > + "Encryption algorithm to use" recommendation ".", > + NULL, NULL, 0, &encryption_algorithm_typelib); > + > +static struct st_mysql_sys_var* settings[] = { > + MYSQL_SYSVAR(vault_url), > + MYSQL_SYSVAR(secret_mount_point), > + MYSQL_SYSVAR(token), > + MYSQL_SYSVAR(vault_ca), > + MYSQL_SYSVAR(encryption_algorithm), > + NULL > +}; > + > +static char curl_errbuf [CURL_ERROR_SIZE]; > + > +static std::string get_error_from_curl (CURLcode curl_code) > +{ > + size_t len = strlen(curl_errbuf); > + std::ostringstream stream; > + if (curl_code != CURLE_OK) > + { > + stream << "CURL returned this error code: " << curl_code; > + stream << " with error message : "; > + if (len) > + stream << curl_errbuf; > + else > + stream << curl_easy_strerror(curl_code); > + } > + return stream.str(); > +} > + > +#define max_response_size 65536 > + > +static size_t write_response_memory(void *contents, size_t size, size_t nmemb, void *userp) > +{ > + size_t realsize = size * nmemb; > + if (size != 0 && realsize / size != nmemb) > + return 0; // overflow > + std::ostringstream *read_data = static_cast<std::ostringstream*>(userp); > + size_t ss_pos = read_data->tellp(); > + read_data->seekp(0, std::ios::end); > + size_t number_of_read_bytes = read_data->tellp(); > + read_data->seekp(ss_pos); > + if (number_of_read_bytes + realsize > max_response_size) > + return 0; // response size limit exceeded > + read_data->write(static_cast<char*>(contents), realsize); > + if (!read_data->good()) > + return 0; > + return realsize; > +} > + > +std::ostringstream read_data_stream; > + > +#define timeout 300 > + > +static struct curl_slist *list = NULL; > + > +static bool setup_curl_session (CURL * curl) > +{ > + std::string token_header = std::string("X-Vault-Token:") + std::string(token); > + CURLcode curl_res = CURLE_OK; > + read_data_stream.str(""); > + read_data_stream.clear(); > + curl_errbuf[0] = '\0'; > + if (list != NULL) > + { > + curl_slist_free_all(list); > + list = NULL; > + } 1. why do you recreate the list? isn't it always the same? 2. there's no protection against concurrent access. this probably won't work at all when many threads will try to request keys in parallel. note, I don't think you need a mutex protection here, better populate the list once in the plugin initializer and then use it. > + if ((list= curl_slist_append(list, token_header.c_str())) == NULL || > + (list= curl_slist_append(list, "Content-Type: application/json")) == > + NULL || > + (curl_res= curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errbuf)) != > + CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, > + write_response_memory)) != CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_WRITEDATA, > + static_cast<void *>(&read_data_stream))) != > + CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list)) != > + CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1)) != > + CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L)) != > + CURLE_OK || > + (vault_ca != NULL && strlen(vault_ca) != 0 && > + (curl_res= curl_easy_setopt(curl, CURLOPT_CAINFO, vault_ca)) != > + CURLE_OK) || > + (curl_res= curl_easy_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_ALL)) != > + CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L)) != > + CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, timeout)) != > + CURLE_OK || > + (curl_res= curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout)) != > + CURLE_OK || > + (curl_res = curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, > + (long)CURL_HTTP_VERSION_1_1)) != > + CURLE_OK) > + { > + my_printf_error(ER_UNKNOWN_ERROR, > + get_error_from_curl(curl_res).c_str(), > + ME_ERROR_LOG_ONLY); > + return true; > + } > + return false; > +} > + > +static bool curl_run (char * url, const char * * response) > +{ > + CURLcode curl_res = CURLE_OK; > + long http_code = 0; > + CURL *curl = curl_easy_init(); the manual for curl_easy_init says If you did not already call curl_global_init(3), curl_easy_init(3) does it automatically. This may be lethal in multi-threaded cases, since curl_global_init(3) is not thread-safe, and it may result in resource problems because there is no corresponding cleanup. You are strongly advised to not allow this automatic behaviour, by calling curl_global_init(3) yourself properly. See the description in libcurl(3) of global environment requirements for details of how to use this function. > + if (curl == NULL) > + { > + my_printf_error(ER_UNKNOWN_ERROR, > + "Cannot initialize curl session", > + ME_ERROR_LOG_ONLY); > + return true; > + } > + if (setup_curl_session(curl) || > + (curl_res = curl_easy_setopt(curl, CURLOPT_URL, url)) != CURLE_OK || > + (curl_res = curl_easy_perform(curl)) != CURLE_OK || > + (curl_res = curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code)) != CURLE_OK) > + { > + my_printf_error(ER_UNKNOWN_ERROR, > + get_error_from_curl(curl_res).c_str(), > + ME_ERROR_LOG_ONLY); > + return true; > + } > + if (http_code == 404) > + { > + *response = NULL; > + return false; > + } > + *response = read_data_stream.str().c_str(); > + return http_code < 200 || http_code >= 300; > +} > + > +static unsigned int get_latest_version (uint key_id) > +{ > + char url[2048]; > + const char *response; > + const char *js, *ver; > + int js_len, ver_len; > + size_t response_len; > + snprintf(url, sizeof(url), "%s/v1/%s/%u", > + vault_url, secret_mount_point, key_id); > + if (curl_run(url, &response)) { > + my_printf_error(ER_UNKNOWN_ERROR, > + "Unable to get key data", > + 0); > + return ENCRYPTION_KEY_VERSION_INVALID; > + } > + response_len = strlen(response); > + if (json_get_object_key(response, response + response_len, > + "metadata", &js, &js_len) != JSV_OBJECT) > + return ENCRYPTION_KEY_VERSION_INVALID; > + if (json_get_object_key(js, js+js_len,"version", &ver, &ver_len) != JSV_NUMBER) > + return ENCRYPTION_KEY_VERSION_INVALID; > + std::string ver_str = std::string(ver, ver_len); > + return strtoul(ver_str.c_str(), NULL, 10); > +} > + > +static int decode_data (const char *src, int src_len, > + unsigned char* dstbuf, unsigned *buflen) > +{ > + int length_of_memory_needed_for_decode = > + my_base64_needed_decoded_length(src_len); > + char* data= (char *)my_malloc( > + length_of_memory_needed_for_decode * sizeof(char), MYF(0)); > + int decoded_length= > + my_base64_decode(src, src_len, data, NULL, 0); > + if (decoded_length <= 0) > + { > + memset(data, 0, length_of_memory_needed_for_decode); > + my_free(data); > + return ENCRYPTION_KEY_BUFFER_TOO_SMALL; > + } > + if (*buflen < (unsigned) decoded_length) { > + *buflen= decoded_length; > + my_free(data); you cannot use my_malloc and my_free here, the plugin won't link. use malloc/free instead. > + return ENCRYPTION_KEY_BUFFER_TOO_SMALL; > + } > + *buflen= decoded_length; > + memcpy(dstbuf, data, decoded_length); > + my_free(data); > + return 0; > +} > + > +static unsigned int get_key_from_vault (unsigned int key_id, > + unsigned int key_version, > + unsigned char* dstbuf, > + unsigned int *buflen) > +{ > + char url[2048]; > + const char *response; > + const char *js, *ver, *key; > + int js_len, key_len, ver_len; > + size_t response_len; > + if (key_version != ENCRYPTION_KEY_VERSION_INVALID) > + snprintf(url, sizeof(url), "%s/v1/%s/%u?%u", > + vault_url, secret_mount_point, key_id, key_version); > + else > + snprintf(url, sizeof(url), "%s/v1/%s/%u", > + vault_url, secret_mount_point, key_id); > + if (curl_run(url, &response)) > + { > + my_printf_error(ER_UNKNOWN_ERROR, > + "Unable to get key data", > + 0); > + return ENCRYPTION_KEY_VERSION_INVALID; > + } > + response_len = strlen(response); > + if (key_version != ENCRYPTION_KEY_VERSION_INVALID) > + { > + if (json_get_object_key(response, response + response_len, > + "metadata", &js, &js_len) != JSV_OBJECT) > + return ENCRYPTION_KEY_VERSION_INVALID; > + if (json_get_object_key(js, js+js_len,"version", &ver, &ver_len) != JSV_NUMBER) > + return ENCRYPTION_KEY_VERSION_INVALID; > + std::string ver_str = std::string(ver, ver_len); > + if (strtoul(ver_str.c_str(), NULL, 10) != key_version) > + return ENCRYPTION_KEY_VERSION_INVALID; Can this happen? You've requested a specific version, can Hashicorp return a _different_ version? That'd be strange. > + } > + if (json_get_object_key(response, response + response_len, > + "data", &js, &js_len) != JSV_OBJECT) > + return ENCRYPTION_KEY_VERSION_INVALID; > + if (json_get_object_key(js, js + js_len, "data", &key, &key_len) != JSV_STRING) > + return ENCRYPTION_KEY_VERSION_INVALID; > + return decode_data(key, key_len, dstbuf, buflen); > +} > + > +// let's simplify the condition below > +#ifndef HAVE_EncryptAes128Gcm > +#define MY_AES_GCM MY_AES_CTR > +#ifndef HAVE_EncryptAes128Ctr > +#define MY_AES_CTR MY_AES_CBC > +#endif > +#endif > + > +static inline enum my_aes_mode mode(int flags) > +{ > + /* > + If encryption_algorithm is AES_CTR then > + if no-padding, use AES_CTR > + else use AES_GCM (like CTR but appends a "checksum" block) > + else > + use AES_CBC > + */ > + if (encryption_algorithm) > + if (flags & ENCRYPTION_FLAG_NOPAD) > + return MY_AES_CTR; > + else > + return MY_AES_GCM; > + else > + return MY_AES_CBC; > +} > + > +static int ctx_init(void *ctx, const unsigned char* key, unsigned int klen, > + const unsigned char* iv, unsigned int ivlen, int flags, > + unsigned int key_id, unsigned int key_version) > +{ > + return my_aes_crypt_init(ctx, mode(flags), flags, key, klen, iv, ivlen); > +} > + > +static int ctx_update(void *ctx, const unsigned char *src, unsigned int slen, > + unsigned char *dst, unsigned int *dlen) > +{ > + return my_aes_crypt_update(ctx, src, slen, dst, dlen); > +} > + > + > +static int ctx_finish(void *ctx, unsigned char *dst, unsigned int *dlen) > +{ > + return my_aes_crypt_finish(ctx, dst, dlen); > +} > + > +static unsigned int get_length(unsigned int slen, unsigned int key_id, > + unsigned int key_version) > +{ > + return my_aes_get_size(mode(0), slen); > +} > + > +static uint ctx_size(uint, uint) > +{ > + return my_aes_ctx_size(mode(0)); > +} > + > +struct st_mariadb_encryption hashicorp_key_management_plugin= { > + MariaDB_ENCRYPTION_INTERFACE_VERSION, > + get_latest_version, > + get_key_from_vault, > + ctx_size, > + ctx_init, > + ctx_update, > + ctx_finish, > + get_length you've seen my template, I've intentionally did not define any functions beyond get_latest_version and get_key_from_vault. You don't need to either, just put 0, don't copy this code from the file plugin. > +}; > + > +static int hashicorp_key_management_plugin_init(void *p) > +{ > + return 0; > +} if you don't need to initialize the plugin - don't create an initializer at all. what's the point in a dummy one? > + > +static int hashicorp_key_management_plugin_deinit(void *p) > +{ > + read_data_stream.str(""); > + read_data_stream.clear(); > + return 0; > +} > + > +/* > + Plugin library descriptor > +*/ > +maria_declare_plugin(hashicorp_key_management) > +{ > + MariaDB_ENCRYPTION_PLUGIN, > + &hashicorp_key_management_plugin, > + "hashicorp_key_management", > + "MariaDB Corporation", you don't want to have your name here? :) > + "HashiCorp Vault key management plugin", > + PLUGIN_LICENSE_GPL, > + hashicorp_key_management_plugin_init, > + hashicorp_key_management_plugin_deinit, > + 0x0100 /* 1.0 */, > + NULL, /* status variables */ > + settings, > + "1.0", > + MariaDB_PLUGIN_MATURITY_STABLE > +} > +maria_declare_plugin_end; Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] systemd changes in Debian Sid
by Faustin Lammler 18 Nov '19

18 Nov '19

Hi Otto! Otto Kekäläinen <otto(a)debian.org>, 11/11/2019 – 15:06:24 (+0100): > Hello Faustin! > > I have been updating the MariaDB 10.3 packaging in Debian to compat level > 12, and that includes some changes to how the systemd services are > installed. Do you have any comments? > > https://salsa.debian.org/mariadb-team/mariadb-10.3/commits/bugfix/systemd-r… See my comment on the d14de1c0 WIP commit. > Are there some systemd issues regarding 10.3 that you think we should put > in while at this? Yes and it is related to all the extra work that is needed to support both mysql and mariadb systemctl commands (see my comment and Ondrej commit referenced in it). IMO we should definitively remove aliases from systemd unit files (also suggested by M. Biebl #932289) but I did not have time to get things to move on (https://github.com/MariaDB/server/pull/1172 or MDEV-15526) and if possible I believe that we should try to upstream it first. So for the moment I have no other changes to suggest in salsa. Faustin

1 0

Re: [Maria-developers] systemd changes in Debian Sid
by Faustin Lammler 12 Nov '19

12 Nov '19

Hi Otto! I did not had time to look at this for now. I believe I will have a bit of time before the week-end and surely next week. Faustin Otto Kekäläinen <otto(a)debian.org>, 11/11/2019 – 15:06:24 (+0100): > Hello Faustin! > > I have been updating the MariaDB 10.3 packaging in Debian to compat level > 12, and that includes some changes to how the systemd services are > installed. Do you have any comments? > > https://salsa.debian.org/mariadb-team/mariadb-10.3/commits/bugfix/systemd-r… > > Are there some systemd issues regarding 10.3 that you think we should put > in while at this? > > - Otto

1 0

Re: [Maria-developers] c790763e1f0: MDEV-19903 Setup default partitions for system versioning
by Sergei Golubchik 06 Nov '19

06 Nov '19

Hi, Aleksey! Looks good! Ok to push On Nov 06, Aleksey Midenkov wrote: > revision-id: c790763e1f0 (mariadb-10.4.7-51-gc790763e1f0) > parent(s): f10c9ac941e > author: Aleksey Midenkov <midenok(a)gmail.com> > committer: Aleksey Midenkov <midenok(a)gmail.com> > timestamp: 2019-08-19 12:00:33 +0300 > message: > > MDEV-19903 Setup default partitions for system versioning > > Implement syntax like: > > create table t1 (x int) with system versioning partition by system_time; Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 4de59fa0f92: MENT-360: ASAN heap-use-after-free in strmake_root / Query_arena::strmake
by Sergei Golubchik 06 Nov '19

06 Nov '19

Hi, Oleksandr! On Nov 06, Oleksandr Byelkin wrote: > revision-id: 4de59fa0f92 (mariadb-10.2.27-129-g4de59fa0f92) > parent(s): 4b21495343d > author: Oleksandr Byelkin <sanja(a)mariadb.com> > committer: Oleksandr Byelkin <sanja(a)mariadb.com> > timestamp: 2019-10-18 15:14:20 +0200 > message: > > MENT-360: ASAN heap-use-after-free in strmake_root / Query_arena::strmake > > Do not allow thd->query point to freed memory, use safe method to set query. > > --- > mysql-test/r/processlist.result | 32 +++++++++++++++++++++++ > mysql-test/t/processlist.test | 58 +++++++++++++++++++++++++++++++++++++++++ > sql/sql_prepare.cc | 7 ++++- > 3 files changed, 96 insertions(+), 1 deletion(-) > > diff --git a/mysql-test/r/processlist.result b/mysql-test/r/processlist.result > index ab518d961ef..066154feaf1 100644 > --- a/mysql-test/r/processlist.result > +++ b/mysql-test/r/processlist.result > @@ -43,3 +43,35 @@ Message Incorrect string value: '\xF0\x9F\x98\x8Eyy...' for column `information_ > # > # End of 10.1 tests > # > +# > +# MENT-360: ASAN heap-use-after-free in strmake_root / > +# Query_arena::strmake > +# > +CREATE PROCEDURE pr1() > +BEGIN > +DECLARE CONTINUE HANDLER FOR 1146 SET @a = 1; > +DECLARE CONTINUE HANDLER FOR 1243 SET @a = 1; > +LOOP > +PREPARE stmt FROM "CREATE TEMPORARY TABLE ps AS SELECT * FROM non_existing_table"; > +EXECUTE stmt; > +END LOOP; > +END $ > +CREATE PROCEDURE pr2() > +BEGIN > +DECLARE CONTINUE HANDLER FOR 1094 SET @a = 1; > +LOOP > +SHOW EXPLAIN FOR 1; > +END LOOP; > +END $ > +connect con1,localhost,root,,; > +CALL pr1(); > +connect con2,localhost,root,,; > +CALL pr2(); > +connection default; > +KILL 6; > +KILL 7; ^^^ you forgot replace_result also, mysqltest was designed for deterministic test cases, it'd be better to rewrite this test to be deterministic, if possible. > +DROP PROCEDURE pr1; > +DROP PROCEDURE pr2; > +# > +# end of Enterprise tests > +# > diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc > index 525f09d611b..d0043979135 100644 > --- a/sql/sql_prepare.cc > +++ b/sql/sql_prepare.cc > @@ -2885,6 +2885,11 @@ void mysql_sql_stmt_prepare(THD *thd) > > if (stmt->prepare(query.str, (uint) query.length)) > { > + /* > + stmt->prepare() sets thd->query_string, so we have to reset it beck, > + before it will point to uninitialised memory > + */ > + thd->set_query(orig_query); > /* Statement map deletes the statement on erase */ > thd->stmt_map.erase(stmt); > } > @@ -2902,7 +2907,7 @@ void mysql_sql_stmt_prepare(THD *thd) > But here we should restore the original query so it's mentioned in > logs properly. > */ > - thd->set_query_inner(orig_query); > + thd->set_query(orig_query); > DBUG_VOID_RETURN; > } may be better to have set_query() only once, like in bool res=stmt->prepare(query.str, (uint) query.length); thd->set_query(orig_query); if (res) ... Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] aeb3432849e: MDEV-20076: SHOW GRANTS does not quote role names properly
by Sergei Golubchik 06 Nov '19

06 Nov '19

Hi, Oleksandr! On Nov 06, Oleksandr Byelkin wrote: > revision-id: aeb3432849e (mariadb-10.3.19-2-gaeb3432849e) > parent(s): 5d3bd2b75b5 > author: Oleksandr Byelkin <sanja(a)mariadb.com> > committer: Oleksandr Byelkin <sanja(a)mariadb.com> > timestamp: 2019-11-06 12:35:19 +0100 > message: > > MDEV-20076: SHOW GRANTS does not quote role names properly > > Quotes added to output. > > +# > +# MDEV-20076: SHOW GRANTS does not quote role names properly > +# > +create role 'role-1'; > +create user 'user-1'@'localhost'; > +grant select on mysql.user to 'role-1'; > +GRANT 'role-1' TO 'user-1'@'localhost'; > +show grants for 'role-1'; > +Grants for role-1 > +GRANT USAGE ON *.* TO 'role-1' > +GRANT SELECT ON `mysql`.`user` TO 'role-1' > +show grants for 'user-1'@'localhost'; > +Grants for user-1@localhost > +GRANT 'role-1' TO 'user-1'@'localhost' > +GRANT USAGE ON *.* TO 'user-1'@'localhost' > +drop role 'role-1'; > +drop user 'user-1'@'localhost'; Please, add a test for names that contains a single quote. Like create role `role'1`; create user `user'1`@localhost; and the rest as in your test above. Or, if you want it to look a bit less artificial create role `rock'n'roll`; create user `O'Brien`@localhost; Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] d5352b8154d: MDEV-20015 Assertion `!in_use->is_error()' failed in TABLE::update_virtual_field
by Sergei Golubchik 31 Oct '19

31 Oct '19

Hi, Aleksey! On Oct 25, Aleksey Midenkov wrote: > revision-id: d5352b8154d (mariadb-10.2.25-54-gd5352b8154d) > parent(s): 1153950ad0a > author: Aleksey Midenkov <midenok(a)gmail.com> > committer: Aleksey Midenkov <midenok(a)gmail.com> > timestamp: 2019-07-22 15:40:06 +0300 > message: > > MDEV-20015 Assertion `!in_use->is_error()' failed in TABLE::update_virtual_field > > Preserve and restore statement DA. This is strange. Diagnostics areas aren't supposed to be temporarily created on the frame, they aren't arenas. Why TABLE::update_virtual_field() is called at all if there's already an error? > diff --git a/sql/table.cc b/sql/table.cc > index f5b5bad99cc..65611d78bde 100644 > --- a/sql/table.cc > +++ b/sql/table.cc > @@ -7682,15 +7682,25 @@ int TABLE::update_virtual_fields(handler *h, enum_vcol_update_mode update_mode) > > int TABLE::update_virtual_field(Field *vf) > { > - DBUG_ASSERT(!in_use->is_error()); > + Diagnostics_area *stmt_da= NULL; > + Diagnostics_area tmp_stmt_da(in_use->query_id, false, true); > + bool error; > Query_arena backup_arena; > DBUG_ENTER("TABLE::update_virtual_field"); > + if (unlikely(in_use->is_error())) > + { > + stmt_da= in_use->get_stmt_da(); > + in_use->set_stmt_da(&tmp_stmt_da); > + } > in_use->set_n_backup_active_arena(expr_arena, &backup_arena); > bitmap_clear_all(&tmp_set); > vf->vcol_info->expr->walk(&Item::update_vcol_processor, 0, &tmp_set); > vf->vcol_info->expr->save_in_field(vf, 0); > in_use->restore_active_arena(expr_arena, &backup_arena); > - DBUG_RETURN(in_use->is_error()); > + error= in_use->is_error(); > + if (stmt_da) > + in_use->set_stmt_da(stmt_da); > + DBUG_RETURN(error); > } Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 3

Re: [Maria-developers] f6269a85ad7: MDEV-17553 Enable setting start datetime for interval partitioned history of system versioned tables
by Sergei Golubchik 31 Oct '19

31 Oct '19

Hi, Aleksey! On Sep 26, Aleksey Midenkov wrote: > revision-id: f6269a85ad7 (mariadb-10.4.7-38-gf6269a85ad7) > parent(s): a9ca752f1a9 > author: Aleksey Midenkov <midenok(a)gmail.com> > committer: Aleksey Midenkov <midenok(a)gmail.com> > timestamp: 2019-08-19 11:58:56 +0300 > message: > > MDEV-17553 Enable setting start datetime for interval partitioned history of system versioned tables > > * Interactive STARTS syntax I wouldn't use the word "interactive" here, better say "Explicit STARTS syntax" > * SHOW CREATE > * Default STARTS rounding depending on INTERVAL type This is questionable. I don't see why it's much better than the old behavior. It kind of makes sense, but that's all. If it would've worked that way from the start, it would be fine. But now, I'm not sure it offers such important benefits to break the compatibility for it. > * Warn when STARTS timestamp is further than INTERVAL > > [Closes tempesta-tech/mariadb#574] > > === Dependency hints (auto-detected by git-deps) === > 336c0139a89 MDEV-16370 row-based binlog events (updates by primary key) can not be applied multiple times to system versioned tables ^^^ remove these dependency hints before pushing, please > diff --git a/mysql-test/suite/versioning/r/partition.result b/mysql-test/suite/versioning/r/partition.result > index 9e532824414..3606c4e407f 100644 > --- a/mysql-test/suite/versioning/r/partition.result > +++ b/mysql-test/suite/versioning/r/partition.result > @@ -1,3 +1,4 @@ > +call mtr.add_suppression("need more HISTORY partitions"); why these warnings suddenly started to show up in the error log? > set system_versioning_alter_history=keep; > # Check conventional partitioning on temporal tables > create or replace table t1 ( > @@ -266,11 +267,11 @@ x > 6 > insert into t1 values (7), (8); > Warnings: > -Warning 4114 Versioned table `test`.`t1`: partition `p1` is full, add more HISTORY partitions > +Warning 4114 Versioned table `test`.`t1`: last HISTORY partition (`p1`) is out of LIMIT, need more HISTORY partitions Hmm, "is out of LIMIT" sounds strange, I liked "is full" more. Is it that important to say LIMIT or INTERVAL here? > ### warn about full partition > delete from t1; > Warnings: > -Warning 4114 Versioned table `test`.`t1`: partition `p1` is full, add more HISTORY partitions > +Warning 4114 Versioned table `test`.`t1`: last HISTORY partition (`p1`) is out of LIMIT, need more HISTORY partitions > select * from t1 partition (p1) order by x; > x > 4 > diff --git a/mysql-test/suite/versioning/r/partition_rotation.result b/mysql-test/suite/versioning/r/partition_rotation.result > index 69b30a56bd6..f6db36b117b 100644 > --- a/mysql-test/suite/versioning/r/partition_rotation.result > +++ b/mysql-test/suite/versioning/r/partition_rotation.result > @@ -55,4 +57,265 @@ i > explain partitions select * from t1 for system_time all where row_end = @ts; > id select_type table partitions type possible_keys key key_len ref rows Extra > 1 SIMPLE t1 p1_p1sp0,p1_p1sp1 # NULL NULL NULL NULL # # > -drop table t1; > +## INTERVAL ... STARTS > +create or replace table t1 (i int) with system versioning > +partition by system_time interval 1 day starts 'a' > +(partition p0 history, partition pn current); > +ERROR HY000: Wrong parameters for partitioned `t1`: wrong value for 'STARTS' > +create or replace table t1 (i int) with system versioning > +partition by system_time interval 1 day starts '00:00:00' > +(partition p0 history, partition pn current); > +ERROR HY000: Wrong parameters for partitioned `t1`: wrong value for 'STARTS' There are clear and well defined rules how to cast TIME to TIMESTAMP, so we can allow that. But it might be confusing, so if you'd like, let's keep it your way and lift this restriction later if needed. > +create or replace table t1 (i int) with system versioning > +partition by system_time interval 1 day starts '2000-00-01 00:00:00' > +(partition p0 history, partition pn current); > +ERROR HY000: Wrong parameters for partitioned `t1`: wrong value for 'STARTS' > +create or replace table t1 (i int) with system versioning > +partition by system_time interval 1 day starts '2000-01-01 00:00:00' > +(partition p0 history, partition pn current); > +show create table t1; > +Table Create Table > +t1 CREATE TABLE `t1` ( > + `i` int(11) DEFAULT NULL > +) ENGINE=MyISAM DEFAULT CHARSET=latin1 WITH SYSTEM VERSIONING > + PARTITION BY SYSTEM_TIME INTERVAL 1 DAY STARTS TIMESTAMP'2000-01-01 00:00:00' > +(PARTITION `p0` HISTORY ENGINE = MyISAM, > + PARTITION `pn` CURRENT ENGINE = MyISAM) > +create or replace table t1 (i int) with system versioning > +partition by system_time interval 1 day starts 946684800 Now, this is very questionable. There is a well defined rule how to cast numbers to temporal values, and this isn't it. If you want to allow numbers, it has to be partition by system_time interval 1 day starts 20000101000000 Whatever the server is doing internally when parsing frms isn't really relevant for the end user, is it? > +(partition p0 history, partition pn current); > +show create table t1; > +Table Create Table > +t1 CREATE TABLE `t1` ( > + `i` int(11) DEFAULT NULL > +) ENGINE=MyISAM DEFAULT CHARSET=latin1 WITH SYSTEM VERSIONING > + PARTITION BY SYSTEM_TIME INTERVAL 1 DAY STARTS TIMESTAMP'2000-01-01 00:00:00' > +(PARTITION `p0` HISTORY ENGINE = MyISAM, > + PARTITION `pn` CURRENT ENGINE = MyISAM) > +# Test STARTS warning > +set timestamp= unix_timestamp('2000-01-01 00:00:00'); > +create or replace table t1 (i int) with system versioning > +partition by system_time interval 1 day > +(partition p0 history, partition pn current); > +show create table t1; > +Table Create Table > +t1 CREATE TABLE `t1` ( > + `i` int(11) DEFAULT NULL > +) ENGINE=MyISAM DEFAULT CHARSET=latin1 WITH SYSTEM VERSIONING > + PARTITION BY SYSTEM_TIME INTERVAL 1 DAY STARTS TIMESTAMP'2000-01-01 00:00:00' > +(PARTITION `p0` HISTORY ENGINE = MyISAM, > + PARTITION `pn` CURRENT ENGINE = MyISAM) > +# no warning > +create or replace table t1 (i int) with system versioning > +partition by system_time interval 1 day starts '2000-01-02 00:00:00' Why no warning here? If I insert and delete a row right now, it'll happen at 2000-01-01 00:00:00. So, it'll be before STARTS, what partition it should go into? As far as I understand, STARTS normally must be before NOW() not before NOW()+INTERVAL > diff --git a/sql/partition_info.cc b/sql/partition_info.cc > index ba6fc8a49ec..119c00dfce6 100644 > --- a/sql/partition_info.cc > +++ b/sql/partition_info.cc > @@ -2381,6 +2383,111 @@ static bool strcmp_null(const char *a, const char *b) > return true; > } > > +/** > + Assign INTERVAL and STARTS for SYSTEM_TIME partitions. > + > + @return true on error > +*/ > + > +bool partition_info::vers_set_interval(THD* thd, Item* interval, > + interval_type int_type, Item* starts, > + const char *table_name) why do you pass the table name as an argument now? > +{ > + DBUG_ASSERT(part_type == VERSIONING_PARTITION); > + > + const bool interactive= !table; what does that mean? * first, SQL is not interactive, using this adjective is confusing, * second you didn't write if(!table) because you wanted to use a self-documenting variable name, I presume. This name isn't helping, it confuses even more :( please, either use a really self-documenting name, or just write if(!table) with a comment. ... okay, now I see what you mean. Better use a comment: if (!table) { /* called from mysql_unpack_partition() not from mysql_parse() */ > + MYSQL_TIME ltime; > + uint err; > + vers_info->interval.type= int_type; > + > + /* 1. assign INTERVAL to interval.step */ > + if (interval->fix_fields_if_needed_for_scalar(thd, &interval)) > + return true; > + bool error= get_interval_value(thd, interval, int_type, &vers_info->interval.step) || > + vers_info->interval.step.neg || vers_info->interval.step.second_part || > + !(vers_info->interval.step.year || vers_info->interval.step.month || > + vers_info->interval.step.day || vers_info->interval.step.hour || > + vers_info->interval.step.minute || vers_info->interval.step.second); > + if (error) { indentation { > + my_error(ER_PART_WRONG_VALUE, MYF(0), table_name, "INTERVAL"); > + return true; > + } > + > + /* 2. assign STARTS to interval.start */ > + if (starts) > + { I've already commented on the following, but here it is again: > + if (starts->fix_fields_if_needed_for_scalar(thd, &starts)) > + return true; > + switch (starts->result_type()) > + { > + case INT_RESULT: > + case DECIMAL_RESULT: > + case REAL_RESULT: > + if (starts->val_int() > TIMESTAMP_MAX_VALUE) > + goto interval_starts_error; > + vers_info->interval.start= (time_t) starts->val_int(); > + break; wrong conversion to temporal > + case STRING_RESULT: > + case TIME_RESULT: > + { > + Datetime::Options opt(TIME_NO_ZERO_DATE | TIME_NO_ZERO_IN_DATE, thd); > + starts->get_date(thd, &ltime, opt); > + vers_info->interval.start= TIME_to_timestamp(thd, &ltime, &err); > + if (err) > + goto interval_starts_error; > + break; > + } > + case ROW_RESULT: > + default: > + goto interval_starts_error; > + } > + if (interactive) > + { > + my_tz_OFFSET0->gmt_sec_to_TIME(&ltime, thd->query_start()); > + if (date_add_interval(thd, &ltime, int_type, vers_info->interval.step)) > + return true; I think this isn't needed > + my_time_t boundary= my_tz_OFFSET0->TIME_to_gmt_sec(&ltime, &err); > + if (vers_info->interval.start > boundary) { > + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, > + ER_PART_STARTS_BEYOND_INTERVAL, > + ER_THD(thd, ER_PART_STARTS_BEYOND_INTERVAL), > + table_name); > + } > + } > + } > + else // calculate default STARTS depending on INTERVAL > + { > + thd->variables.time_zone->gmt_sec_to_TIME(&ltime, thd->query_start()); > + if (vers_info->interval.step.second) > + goto interval_set_starts; > + ltime.second= 0; I think this isn't needed > + if (vers_info->interval.step.minute) > + goto interval_set_starts; > + ltime.minute= 0; > + if (vers_info->interval.step.hour) > + goto interval_set_starts; > + ltime.hour= 0; > + if (vers_info->interval.step.day) > + goto interval_set_starts; > + ltime.day= 1; > + if (vers_info->interval.step.month) > + goto interval_set_starts; > + ltime.month= 1; > + DBUG_ASSERT(vers_info->interval.step.year); > + > +interval_set_starts: > + vers_info->interval.start= TIME_to_timestamp(thd, &ltime, &err); > + if (err) > + goto interval_starts_error; > + } > + > + return false; > + > +interval_starts_error: > + my_error(ER_PART_WRONG_VALUE, MYF(0), table_name, "STARTS"); > + return true; > +} > + Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 1

Re: [Maria-developers] 02362a77d0d: MDEV-19848 Server crashes in check_vcol_forward_refs upon INSERT DELAYED into table with long blob key
by Sergei Golubchik 29 Oct '19

29 Oct '19

Hi, Sachin! Looks good now, thanks. See comments below On Oct 29, Sachin Setiya wrote: > revision-id: 02362a77d0d (mariadb-10.4.7-111-g02362a77d0d) > parent(s): 52f3829b95c > author: Sachin <sachin.setiya(a)mariadb.com> > committer: Sachin <sachin.setiya(a)mariadb.com> > timestamp: 2019-10-14 16:10:54 +0530 > message: > > MDEV-19848 Server crashes in check_vcol_forward_refs upon INSERT DELAYED into table with long blob key > > Problem:- Insert delayed is not working with Long Unique Index. > It is failing with > 1. INSERT DELAYED INTO t1 VALUES(); > 2. INSERT DELAYED INTO t1 VALUES(1); > 3. Potential Race condition When Insert DELAYED gets dup key error(After fix), > And it will change original table key_info by calling > re_setup_keyinfo_hash, And second thread is in check_duplicate_long_entries > 4. Insert delayed into INVISIBLE COLUMN will also not work. > > There are 4 main issue > > 1. while calling make_new_field we forgot to & LONG_UNIQUE_HASH_FIELD > flag into new field flags. > > 2. New field created created into get_local_table by make_new_field does > not respect old field visibility, Assigning old field visibility will > solve Problem 4 and part of problem 2. > > 3. As we know Problem 3 race condition is caused because table and > delayed table share same key_info, So we will make a copy of original table > key_info in get_local_table. > > 4. In parse_vcol_defs we have this code block > keypart->field->vcol_info= > table->field[keypart->field->field_index]->vcol_info; > Which is wrong because we should not change original > table->field->vcol_info with vcol_info which is create on delayed > thread. > > --- > mysql-test/main/long_unique_bugs.result | 29 +++++++++++++++++++++++++++++ > mysql-test/main/long_unique_bugs.test | 32 ++++++++++++++++++++++++++++++++ > sql/field.cc | 7 ++++++- > sql/sql_insert.cc | 31 +++++++++++++++++++++++++++++++ > sql/table.cc | 31 ++++++++++++++++++++++++++++++- > sql/table.h | 2 ++ > 6 files changed, 130 insertions(+), 2 deletions(-) > > diff --git a/mysql-test/main/long_unique_bugs.test b/mysql-test/main/long_unique_bugs.test > index 13a4e1367a0..365393aa2ea 100644 > --- a/mysql-test/main/long_unique_bugs.test > +++ b/mysql-test/main/long_unique_bugs.test > @@ -340,3 +340,35 @@ while ($count) > --eval $insert_stmt > --enable_query_log > drop table t1; > +# > +# MDEV-19848 Server crashes in check_vcol_forward_refs upon INSERT DELAYED into table with long blob key > +# > +CREATE TABLE t1 (a blob, UNIQUE(a)) ENGINE=MyISAM; > +INSERT DELAYED t1 () VALUES (1); > +INSERT t1 () VALUES (2); > +# Cleanup > +DROP TABLE t1; > +CREATE TABLE t1 (a char(50), UNIQUE(a(10)) USING HASH); > +INSERT DELAYED t1 () VALUES (1); > +INSERT t1 () VALUES (2); > +# Cleanup > +DROP TABLE t1; > +CREATE TABLE t1 ( > + a CHAR(128), > + b CHAR(128) AS (a), > + c varchar(5000), > + UNIQUE(c,b(64)) > +) ENGINE=myisam; > +INSERT DELAYED t1 (a,c) VALUES (1,1); > +--sleep 1 > +INSERT t1 (a,c) VALUES (2,2); > +INSERT t1 (a,c) VALUES (3,3); > +drop table t1; > +create table t1(a int , b int invisible); > +insert into t1 values(1); > +insert delayed into t1(a,b) values(2,2); > +--echo #Should not fails "Should not fail" > +insert delayed into t1 values(2); I suspect there's a race condition here. You cannot know that INSERT DELAYED will insert rows before you do your SELECT. Other tests usually do FLUSH TABLE t1 before SELECT. Alternatively you can use include/wait_condition.inc file to wait for, say, SELECT COUNT(*) = 3 > +select a,b from t1 order by a; > +# Cleanup > +DROP TABLE t1; > diff --git a/sql/field.cc b/sql/field.cc > index 0eb53f40a54..d1187237f23 100644 > --- a/sql/field.cc > +++ b/sql/field.cc > @@ -2373,8 +2373,13 @@ Field *Field::make_new_field(MEM_ROOT *root, TABLE *new_table, > tmp->flags&= (NOT_NULL_FLAG | BLOB_FLAG | UNSIGNED_FLAG | > ZEROFILL_FLAG | BINARY_FLAG | ENUM_FLAG | SET_FLAG | > VERS_SYS_START_FLAG | VERS_SYS_END_FLAG | > - VERS_UPDATE_UNVERSIONED_FLAG); > + VERS_UPDATE_UNVERSIONED_FLAG | LONG_UNIQUE_HASH_FIELD); > tmp->reset_fields(); > + /* > + Calling make_new_field will return a VISIBLE field, If caller function > + wants original visibility he should change it later. "visibility it should change" > + This is done because view created on invisible fields are visible. "because invisible fields explicitly named in a view become visible" > + */ > tmp->invisible= VISIBLE; > return tmp; > } > diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc > index fe6c5fa8ec4..d80044ec37d 100644 > --- a/sql/sql_insert.cc > +++ b/sql/sql_insert.cc > @@ -2607,6 +2607,11 @@ TABLE *Delayed_insert::get_local_table(THD* client_thd) > { > if (!(*field= (*org_field)->make_new_field(client_thd->mem_root, copy, 1))) > goto error; > + /* > + We want same visibility as of original table because we are just creating > + a clone for delayed insert. > + */ > + (*field)->invisible= (*org_field)->invisible; I'm surprised it matters. I'd thought that all sanity checks, and in particular whether number of columns matches the number of values, are done in the connection user thread before submitting the job to the delayed insert thread. > (*field)->unireg_check= (*org_field)->unireg_check; > (*field)->orig_table= copy; // Remove connection > (*field)->move_field_offset(adjust_ptrs); // Point at copy->record[0] > @@ -2621,7 +2626,33 @@ TABLE *Delayed_insert::get_local_table(THD* client_thd) > if (share->virtual_fields || share->default_expressions || > share->default_fields) > { > + /* > + If we have long unique table then delayed insert can modify key structure > + (re/setup_keyinfo_hash_all) of original table when it gets insert error, > + parse_vcol_defs will also modify key_info structure. So it is better to > + clone the table->key_info for copy table. > + We will not be cloning key_part_info or even changing any field ptr. > + Because re/setup_keyinfo_hash_all only modify key_info array. So it will > + be like having new key_info array for copy table with old key_part_info > + ptr. > + */ > + if (share->long_unique_table) > + { > + KEY *key_info; > + if (!(key_info= (KEY*) client_thd->alloc(share->keys*sizeof(KEY)))) > + goto error; > + copy->key_info= key_info; > + memcpy(key_info, table->key_info, sizeof(KEY)*share->keys); > + } > + /* > + parse_vcol_defs expects key_infos to be in user defined format. > + */ > + copy->setup_keyinfo_hash_all(); > bool error_reported= FALSE; > + /* > + We won't be calling re_setup_keyinfo_hash because parse_vcol_defs changes > + key_infos to storage engine format > + */ > if (unlikely(parse_vcol_defs(client_thd, client_thd->mem_root, copy, > &error_reported, > VCOL_INIT_DEPENDENCY_FAILURE_IS_WARNING))) > diff --git a/sql/table.cc b/sql/table.cc > index 718d0dce072..d0fcd449098 100644 > --- a/sql/table.cc > +++ b/sql/table.cc > @@ -1223,7 +1223,16 @@ bool parse_vcol_defs(THD *thd, MEM_ROOT *mem_root, TABLE *table, > new (mem_root) Item_field(thd, keypart->field), > new (mem_root) Item_int(thd, length)); > list_item->fix_fields(thd, NULL); > - keypart->field->vcol_info= > + /* > + Do not change the vcol_info when vcol_info->expr is not NULL > + This will happen in the case of Delayed_insert::get_local_table() > + And if we change the vcol_info in Delayed insert , then original > + table field->vcol_info will be created on delayed insert thread > + mem_root. > + */ > + if (!keypart->field->vcol_info || > + !keypart->field->vcol_info->expr) > + keypart->field->vcol_info= > table->field[keypart->field->field_index]->vcol_info; > } > else > @@ -9084,6 +9093,26 @@ void re_setup_keyinfo_hash(KEY *key_info) > key_info->ext_key_parts= 1; > key_info->flags&= ~HA_NOSAME; > } > + > +/* > + call setup_keyinfo_hash for all keys in table > + */ > +void TABLE::setup_keyinfo_hash_all() > +{ > + for (uint i= 0; i < s->keys; i++) > + if (key_info[i].algorithm == HA_KEY_ALG_LONG_HASH) > + setup_keyinfo_hash(&key_info[i]); also... I remember you wanted to rename these methods to something more descriptive. > +} > + > +/* > + call re_setup_keyinfo_hash for all keys in table > + */ > +void TABLE::re_setup_keyinfo_hash_all() don't add this method, you aren't using it anywhere > +{ > + for (uint i= 0; i < s->keys; i++) > + if (key_info[i].algorithm == HA_KEY_ALG_LONG_HASH) > + re_setup_keyinfo_hash(&key_info[i]); > +} > /** > @brief clone of current handler. > Creates a clone of handler used in update for Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 96569b793a3: MDEV-19761 - Before Trigger not processed for Not Null Column
by Sergei Golubchik 29 Oct '19

29 Oct '19

Hi, Anel! On Oct 29, Anel Husakovic wrote: > revision-id: 96569b793a3 (mariadb-10.1.39-121-g96569b793a3) > parent(s): e32f29b7f31 > author: Anel Husakovic <anel(a)mariadb.org> > committer: Anel Husakovic <anel(a)mariadb.org> > timestamp: 2019-07-24 23:50:29 -0700 > message: > > MDEV-19761 - Before Trigger not processed for Not Null Column here you should have a commit comment explaining the bug and your solution. > --- > mysql-test/r/trigger.result | 6 +++- > mysql-test/r/trigger_no_defaults-11698.result | 29 +++++++++++++++++++ > mysql-test/r/trigger_null-8605.result | 2 +- > mysql-test/suite/funcs_1/r/innodb_trig_09.result | 4 +-- > mysql-test/suite/funcs_1/r/memory_trig_09.result | 4 +-- > mysql-test/suite/funcs_1/r/myisam_trig_09.result | 4 +-- > mysql-test/t/trigger_no_defaults-11698.test | 37 ++++++++++++++++++++---- > sql/sql_base.cc | 23 +++++++++++++++ > sql/sql_trigger.cc | 2 +- > sql/sql_trigger.h | 4 +++ > 10 files changed, 97 insertions(+), 18 deletions(-) > > diff --git a/mysql-test/r/trigger.result b/mysql-test/r/trigger.result > index 8852a622251..04bb9d9be68 100644 > --- a/mysql-test/r/trigger.result > +++ b/mysql-test/r/trigger.result > @@ -366,6 +366,8 @@ create trigger trg1 before update on t1 for each row set @a:= @a + new.j - old.j > create trigger trg2 after update on t1 for each row set @b:= "Fired"; > set @a:= 0, @b:= ""; > update t1, t2 set j = j + 10 where t1.i = t2.i; > +Warnings: > +Warning 1048 Column 'k' cannot be null this looks wrong. Why there's suddenly a warning here? > select @a, @b; > @a @b > 10 Fired > @@ -1235,11 +1237,13 @@ insert into t1 values (1,1), (2,2), (3,3); > create trigger t1_bu before update on t1 for each row > set new.j = new.j + 10; > update t1 set i= i+ 10 where j > 2; > +Warnings: > +Warning 1048 Column 'j' cannot be null and here > select * from t1; > diff --git a/sql/sql_base.cc b/sql/sql_base.cc > index e8bdff8b48f..d9fe37c83da 100644 > --- a/sql/sql_base.cc > +++ b/sql/sql_base.cc > @@ -8959,6 +8959,29 @@ void switch_to_nullable_trigger_fields(List<Item> &items, TABLE *table) > > while ((item= it++)) > item->walk(&Item::switch_to_nullable_fields_processor, 1, (uchar*)field); > + uint16 cnt=0; > + uchar *nptr; > + nptr= (uchar*)alloc_root(&table->mem_root, (table->s->fields - table->s->null_fields + 7)/8); > + // First find null_ptr for NULL field in case of mixed NULL and NOT NULL fields > + for (Field **f= field; *f; f++) > + { > + if (table->field[cnt]->null_ptr) > + { > + nptr= table->field[cnt]->null_ptr; > + break; > + } > + } > + for (Field **f= field; *f; f++) > + { > + if (!table->field[cnt]->null_ptr) > + { > + (*f)->null_bit= 1<<(cnt+1); > + (*f)->flags&= ~(NOT_NULL_FLAG); > + (*f)->null_ptr= nptr; > + } > + cnt++; > + } > + bzero(nptr, (table->s->fields - table->s->null_fields + 7)/8); what are you doing here? > table->triggers->reset_extra_null_bitmap(); > } > } > diff --git a/sql/sql_trigger.cc b/sql/sql_trigger.cc > index 4ecd8139921..7de2fde0e5c 100644 > --- a/sql/sql_trigger.cc > +++ b/sql/sql_trigger.cc > @@ -1118,7 +1118,7 @@ bool Table_triggers_list::prepare_record_accessors(TABLE *table) > > f->flags= (*fld)->flags; > f->null_ptr= null_ptr; > - f->null_bit= null_bit; > + f->null_bit= (*fld)->null_bit; > if (null_bit == 128) > null_ptr++, null_bit= 1; > else > diff --git a/sql/sql_trigger.h b/sql/sql_trigger.h > index f451dfda1ee..c1174bfd36e 100644 > --- a/sql/sql_trigger.h > +++ b/sql/sql_trigger.h > @@ -226,6 +226,10 @@ class Table_triggers_list: public Sql_alloc > trigger_table->s->null_fields + 7)/8; > bzero(extra_null_bitmap, null_bytes); > } > + uchar *get_extra_null_bitmap() const > + { > + return extra_null_bitmap; > + } This is not used anywhere > private: > bool prepare_record_accessors(TABLE *table); > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] e752eed5354: MDEV-19751 Wrong partitioning by KEY() after key dropped
by Sergei Golubchik 29 Oct '19

29 Oct '19

Hi, Aleksey! On Oct 29, Aleksey Midenkov wrote: > revision-id: e752eed5354 (mariadb-10.4.7-47-ge752eed5354) > parent(s): 8403148452a > author: Aleksey Midenkov <midenok(a)gmail.com> > committer: Aleksey Midenkov <midenok(a)gmail.com> > timestamp: 2019-08-19 12:00:13 +0300 > message: > > MDEV-19751 Wrong partitioning by KEY() after key dropped > > Empty partition by key() clause implies that key might be changed by > ALTER, but inplace algorithm must not be used since repartitioning is > required for a different key. good and very clear comment. > diff --git a/sql/sql_partition.cc b/sql/sql_partition.cc > index 10b9c74d868..d76ff5e2e76 100644 > --- a/sql/sql_partition.cc > +++ b/sql/sql_partition.cc > @@ -5903,6 +5903,45 @@ the generated partition syntax in a correct manner. > *partition_changed= TRUE; > } > } > + /* > + Prohibit inplace when key takes part in partitioning expression > + and is altered (dropped). > + */ > + if (!*partition_changed && tab_part_info->part_field_array) > + { > + KEY *key_info= table->key_info; > + List_iterator_fast<Alter_drop> drop_it(alter_info->drop_list); > + for (uint key= 0; key < table->s->keys; key++, key_info++) > + { > + if (key_info->flags & HA_INVISIBLE_KEY) > + continue; > + const char *key_name= key_info->name.str; > + const Alter_drop *drop; > + drop_it.rewind(); > + while ((drop= drop_it++)) > + { > + if (drop->type == Alter_drop::KEY && > + 0 == my_strcasecmp(system_charset_info, key_name, drop->name)) > + break; > + } > + if (!drop) > + continue; > + for (uint kp= 0; kp < key_info->user_defined_key_parts; ++kp) > + { > + const KEY_PART_INFO &key_part= key_info->key_part[kp]; > + for (Field **part_field= tab_part_info->part_field_array; > + *part_field; ++part_field) > + { > + if (*part_field == key_part.field) > + { > + *partition_changed= TRUE; > + goto search_finished; > + } > + } // for (part_field) > + } // for (key_part) > + } // for (key_info) Hmm, if I read it correctly, you iterate over all key parts of every dropped key and see if this field is also part of the partitioning expression. This sounds kind of strange. What if there are two indexes that use the column `a`, say CREATE TABLE t1 (a int, b int, index(a), index(b,a)) and you partition by key(a). Dropping the second index does not change the partitioning, does it? On the other hand, if it's ALTER TABLE ... MODIFY `a` then it can change partitioning, even when the key hasn't changed, right? So, I suspect your check is too strong in some cases and too loose is others. > + search_finished:; > + } > } > if (thd->work_part_info) > { > diff --git a/storage/connect/mysql-test/connect/r/part_file.result b/storage/connect/mysql-test/connect/r/part_file.result > index 3dabd946b50..480de51cce6 100644 > --- a/storage/connect/mysql-test/connect/r/part_file.result > +++ b/storage/connect/mysql-test/connect/r/part_file.result > @@ -308,12 +308,19 @@ EXPLAIN PARTITIONS SELECT * FROM t1 WHERE id = 10; > id select_type table partitions type possible_keys key key_len ref rows Extra > 1 SIMPLE t1 1 ref XID XID 4 const 1 > DROP INDEX XID ON t1; > +Warnings: > +Warning 1105 Data repartition in 1 is unchecked > +Warning 1105 Data repartition in 2 is unchecked > +Warning 1105 Data repartition in 3 is unchecked why did the result file change? could you add a short explanation of it to the commit comment, please? Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

[Maria-developers] Please review MDEV-18319 BIGINT UNSIGNED Performance issue
by Alexander Barkov 29 Oct '19

29 Oct '19

Hi Sergei, Please review a patch for MDEV-18319 https://github.com/MariaDB/server/commit/8a990ad17746927c6d395ec755a262eda5… Thanks

2 1

Re: [Maria-developers] 7a331ec820b: MDEV-18244 Server crashes in ha_innobase::update_thd / ... / ha_partition::update_next_auto_inc_val.
by Sergei Golubchik 25 Oct '19

25 Oct '19

Hi, Alexey! On Oct 25, Alexey Botchkov wrote: > revision-id: 7a331ec820b (mariadb-10.3.18-65-g7a331ec820b) > parent(s): 716d396bb3b > author: Alexey Botchkov <holyfoot(a)mariadb.com> > committer: Alexey Botchkov <holyfoot(a)mariadb.com> > timestamp: 2019-10-22 01:40:48 +0400 > message: > > MDEV-18244 Server crashes in ha_innobase::update_thd / ... / ha_partition::update_next_auto_inc_val. > > Autoincrement calculation should check if tables were opened. > > --- > mysql-test/main/partition_innodb.result | 13 +++++++++++++ > mysql-test/main/partition_innodb.test | 15 +++++++++++++++ > sql/ha_partition.cc | 2 ++ > 3 files changed, 30 insertions(+) > > diff --git a/mysql-test/main/partition_innodb.result b/mysql-test/main/partition_innodb.result > index f3d24347ff9..93a73a785fe 100644 > --- a/mysql-test/main/partition_innodb.result > +++ b/mysql-test/main/partition_innodb.result > @@ -1028,5 +1028,18 @@ COUNT(*) > 2 > DROP TABLE t1; > # > +# MDEV-18244 Server crashes in ha_innobase::update_thd / ... / ha_partition::update_next_auto_inc_val > +# > +CREATE TABLE t1 (a INT) > +ENGINE=InnoDB > +PARTITION BY RANGE (a) ( > +PARTITION p0 VALUES LESS THAN (6), > +PARTITION pn VALUES LESS THAN MAXVALUE > +); > +INSERT INTO t1 VALUES (4),(5),(6); > +ALTER TABLE t1 MODIFY a INT AUTO_INCREMENT PRIMARY KEY; > +UPDATE t1 PARTITION (p0) SET a = 3 WHERE a = 5; > +DROP TABLE t1; > +# > # End of 10.3 tests > # > diff --git a/mysql-test/main/partition_innodb.test b/mysql-test/main/partition_innodb.test > index 629bc29e758..1e0af06b4da 100644 > --- a/mysql-test/main/partition_innodb.test > +++ b/mysql-test/main/partition_innodb.test > @@ -1105,6 +1105,21 @@ INSERT INTO t1 VALUES (1, 7, 8, 9), (2, NULL, NULL, NULL), (3, NULL, NULL, NULL) > SELECT COUNT(*) FROM t1 WHERE x IS NULL AND y IS NULL AND z IS NULL; > DROP TABLE t1; > > +--echo # > +--echo # MDEV-18244 Server crashes in ha_innobase::update_thd / ... / ha_partition::update_next_auto_inc_val > +--echo # > + > +CREATE TABLE t1 (a INT) > + ENGINE=InnoDB > + PARTITION BY RANGE (a) ( > + PARTITION p0 VALUES LESS THAN (6), > + PARTITION pn VALUES LESS THAN MAXVALUE > + ); > +INSERT INTO t1 VALUES (4),(5),(6); > +ALTER TABLE t1 MODIFY a INT AUTO_INCREMENT PRIMARY KEY; > +UPDATE t1 PARTITION (p0) SET a = 3 WHERE a = 5; > +DROP TABLE t1; > + > --echo # > --echo # End of 10.3 tests > --echo # > diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc > index 5a78249644d..e461532db1a 100644 > --- a/sql/ha_partition.cc > +++ b/sql/ha_partition.cc > @@ -8174,6 +8174,8 @@ int ha_partition::info(uint flag) > ("checking all partitions for auto_increment_value")); > do > { > + if (!bitmap_is_set(&m_opened_partitions, (uint)(file_array - m_file))) > + continue; > file= *file_array; > file->info(HA_STATUS_AUTO | no_lock_flag); > set_if_bigger(auto_increment_value, It seems that the intention was to find the max auto_increment_value over all partitions. If you skip some, it won't be max anymore. On the other hand, I don't see why UPDATE should get the max auto-inc of all partitions. May be the correct fix would be not to do it in UPDATE? Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 52f3829b95c: MDEV-20001 Potential dangerous regression: INSERT INTO >=100 rows fail for myisam table with HASH indexes
by Sergei Golubchik 25 Oct '19

25 Oct '19

Hi, Sachin! This is ok to push. But, still, about Aria. When you push the fix for MDEV-18791, it will be very easy to forget this one. May be you add the same test case for Aria now? It'll fail with something like "not supported" error, which is fine, just use --error. But when you fix MDEV-18791 this test will be enabled and it'll ensure you won't forget to fix Aria for bulk inserts. On Oct 25, Sachin Setiya wrote: > revision-id: 52f3829b95c (mariadb-10.4.7-110-g52f3829b95c) > parent(s): 93d2211e02f > author: Sachin <sachin.setiya(a)mariadb.com> > committer: Sachin <sachin.setiya(a)mariadb.com> > timestamp: 2019-10-09 21:17:32 +0530 > message: > > MDEV-20001 Potential dangerous regression: INSERT INTO >=100 rows fail for myisam table with HASH indexes > > Problem:- > > So the issue is when we do bulk insert with rows > > MI_MIN_ROWS_TO_DISABLE_INDEXES(100) , We try to disable the indexes to > speedup insert. But current logic also disables the long unique indexes. > > Solution:- In ha_myisam::start_bulk_insert if we find long hash index > (HA_KEY_ALG_LONG_HASH) we will not disable the index. > > This commit also refactors the mi_disable_indexes_for_rebuild function, > Since this is function is called at only one place, it is inlined into > start_bulk_insert > > mi_clear_key_active is added into myisamdef.h because now it is also used > in ha_myisam.cc file. > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] a0e3bd09251: Part1: MDEV-20837 Add MariaDB_FUNCTION_PLUGIN
by Sergei Golubchik 16 Oct '19

16 Oct '19

Hi, Alexander! Just a couple of comments, see below: On Oct 16, Alexander Barkov wrote: > revision-id: a0e3bd09251 (mariadb-10.4.4-419-ga0e3bd09251) > parent(s): 22b645ef529 > author: Alexander Barkov <bar(a)mariadb.com> > committer: Alexander Barkov <bar(a)mariadb.com> > timestamp: 2019-10-16 16:26:29 +0400 > message: > > Part1: MDEV-20837 Add MariaDB_FUNCTION_PLUGIN > > - Defining MariaDB_FUNCTION_PLUGIN > - Changing the code in /plugins/type_inet/ and /plugins/type_test/ > to use MariaDB_FUNCTION_PLUGIN instead of MariaDB_FUNCTION_COLLECTION_PLUGIN. > diff --git a/include/mysql/plugin_function.h b/include/mysql/plugin_function.h > new file mode 100644 > index 00000000000..4ce416612e9 > --- /dev/null > +++ b/include/mysql/plugin_function.h > @@ -0,0 +1,58 @@ > +#ifndef MARIADB_PLUGIN_FUNCTION_INCLUDED > +#define MARIADB_PLUGIN_FUNCTION_INCLUDED > +/* Copyright (C) 2019, Alexander Barkov and MariaDB > + > + This program is free software; you can redistribute it and/or modify > + it under the terms of the GNU General Public License as published by > + the Free Software Foundation; version 2 of the License. > + > + This program is distributed in the hope that it will be useful, > + but WITHOUT ANY WARRANTY; without even the implied warranty of > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > + GNU General Public License for more details. > + > + You should have received a copy of the GNU General Public License > + along with this program; if not, write to the Free Software > + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ > + > +/** > + @file > + > + Function Plugin API. > + > + This file defines the API for server plugins that manage functions. > +*/ > + > +#ifdef __cplusplus > + > +#include <mysql/plugin.h> > + > +/* > + API for function plugins. (MariaDB_FUNCTION_PLUGIN) > +*/ > +#define MariaDB_FUNCTION_INTERFACE_VERSION (MYSQL_VERSION_ID << 8) > + > + > +class Plugin_function > +{ > + int m_interface_version; > + Create_func *m_builder; > +public: > + Plugin_function(int interface_version, Create_func *builder) > + :m_interface_version(interface_version), > + m_builder(builder) > + { } Why do you need this ^^^ constructor? > + Plugin_function(Create_func *builder) > + :m_interface_version(MariaDB_FUNCTION_INTERFACE_VERSION), > + m_builder(builder) > + { } > + Create_func *create_func() > + { > + return m_builder; > + } > +}; > + > + > +#endif /* __cplusplus */ > + > +#endif /* MARIADB_PLUGIN_FUNCTION_INCLUDED */ > diff --git a/plugin/type_inet/mysql-test/type_inet/func_inet_plugin.result b/plugin/type_inet/mysql-test/type_inet/func_inet_plugin.result > index 4663ae485e2..a9422f2e4fd 100644 > --- a/plugin/type_inet/mysql-test/type_inet/func_inet_plugin.result > +++ b/plugin/type_inet/mysql-test/type_inet/func_inet_plugin.result > @@ -15,14 +16,94 @@ PLUGIN_LICENSE, > PLUGIN_MATURITY, > PLUGIN_AUTH_VERSION > FROM INFORMATION_SCHEMA.PLUGINS > -WHERE PLUGIN_TYPE='FUNCTION COLLECTION' > - AND PLUGIN_NAME='func_inet'; > -PLUGIN_NAME func_inet > +WHERE PLUGIN_TYPE='FUNCTION' > + AND PLUGIN_NAME IN > +('inet_aton', > +'inet_ntoa', > +'inet6_aton', > +'inet6_ntoa', > +'is_ipv4', > +'is_ipv6', > +'is_ipv4_compat', > +'is_ipv4_mapped') > +ORDER BY PLUGIN_NAME; > +---- ---- > +PLUGIN_NAME inet6_aton > PLUGIN_VERSION 1.0 > PLUGIN_STATUS ACTIVE > -PLUGIN_TYPE FUNCTION COLLECTION > +PLUGIN_TYPE FUNCTION > PLUGIN_AUTHOR MariaDB Corporation > -PLUGIN_DESCRIPTION Function collection test > +PLUGIN_DESCRIPTION Function INET6_ATON() > +PLUGIN_LICENSE GPL > +PLUGIN_MATURITY Experimental INET* functions should probably be Alpha, not Experimental. You expect them to be GA one day, don't you? > +PLUGIN_AUTH_VERSION 1.0 > +---- ---- > +PLUGIN_NAME inet6_ntoa > +PLUGIN_VERSION 1.0 > +PLUGIN_STATUS ACTIVE > +PLUGIN_TYPE FUNCTION > +PLUGIN_AUTHOR MariaDB Corporation > +PLUGIN_DESCRIPTION Function INET6_NTOA() > +PLUGIN_LICENSE GPL > +PLUGIN_MATURITY Experimental > +PLUGIN_AUTH_VERSION 1.0 > +---- ---- > +PLUGIN_NAME inet_aton > +PLUGIN_VERSION 1.0 > +PLUGIN_STATUS ACTIVE > +PLUGIN_TYPE FUNCTION > +PLUGIN_AUTHOR MariaDB Corporation > +PLUGIN_DESCRIPTION Function INET_ATON() > +PLUGIN_LICENSE GPL > +PLUGIN_MATURITY Experimental > +PLUGIN_AUTH_VERSION 1.0 > +---- ---- > +PLUGIN_NAME inet_ntoa > +PLUGIN_VERSION 1.0 > +PLUGIN_STATUS ACTIVE > +PLUGIN_TYPE FUNCTION > +PLUGIN_AUTHOR MariaDB Corporation > +PLUGIN_DESCRIPTION Function INET_NTOA() > +PLUGIN_LICENSE GPL > +PLUGIN_MATURITY Experimental > +PLUGIN_AUTH_VERSION 1.0 > +---- ---- > +PLUGIN_NAME is_ipv4 > +PLUGIN_VERSION 1.0 > +PLUGIN_STATUS ACTIVE > +PLUGIN_TYPE FUNCTION > +PLUGIN_AUTHOR MariaDB Corporation > +PLUGIN_DESCRIPTION Function IS_IPV4() > +PLUGIN_LICENSE GPL > +PLUGIN_MATURITY Experimental > +PLUGIN_AUTH_VERSION 1.0 > +---- ---- > +PLUGIN_NAME is_ipv4_compat > +PLUGIN_VERSION 1.0 > +PLUGIN_STATUS ACTIVE > +PLUGIN_TYPE FUNCTION > +PLUGIN_AUTHOR MariaDB Corporation > +PLUGIN_DESCRIPTION Function IS_IPV4_COMPAT() > +PLUGIN_LICENSE GPL > +PLUGIN_MATURITY Experimental > +PLUGIN_AUTH_VERSION 1.0 > +---- ---- > +PLUGIN_NAME is_ipv4_mapped > +PLUGIN_VERSION 1.0 > +PLUGIN_STATUS ACTIVE > +PLUGIN_TYPE FUNCTION > +PLUGIN_AUTHOR MariaDB Corporation > +PLUGIN_DESCRIPTION Function IS_IPV4_MAPPED() > +PLUGIN_LICENSE GPL > +PLUGIN_MATURITY Experimental > +PLUGIN_AUTH_VERSION 1.0 > +---- ---- > +PLUGIN_NAME is_ipv6 > +PLUGIN_VERSION 1.0 > +PLUGIN_STATUS ACTIVE > +PLUGIN_TYPE FUNCTION > +PLUGIN_AUTHOR MariaDB Corporation > +PLUGIN_DESCRIPTION Function IS_IPV6() > PLUGIN_LICENSE GPL > PLUGIN_MATURITY Experimental > PLUGIN_AUTH_VERSION 1.0 > diff --git a/sql/item_create.cc b/sql/item_create.cc > index e8eb76dfc12..e316723cedf 100644 > --- a/sql/item_create.cc > +++ b/sql/item_create.cc > @@ -5704,12 +5657,26 @@ void item_create_cleanup() > { > DBUG_ENTER("item_create_cleanup"); > my_hash_free(& native_functions_hash); > -#ifdef HAVE_SPATIAL > - plugin_function_collection_geometry.deinit(); > -#endif > DBUG_VOID_RETURN; > } > > + > +static Create_func * > +function_plugin_find_native_function_builder(THD *thd, const LEX_CSTRING &name) > +{ > + plugin_ref plugin; > + if ((plugin= my_plugin_lock_by_name(thd, &name, MariaDB_FUNCTION_PLUGIN))) > + { > + Create_func *builder= > + reinterpret_cast<Plugin_function*>(plugin_decl(plugin)->info)-> > + create_func(); > + plugin_unlock(thd, plugin); > + return builder; > + } > + return NULL; So, function plugins cannot be unlocked either? > +} > + > + > Create_func * > find_native_function_builder(THD *thd, const LEX_CSTRING *name) > { > @@ -5724,16 +5691,10 @@ find_native_function_builder(THD *thd, const LEX_CSTRING *name) > if (func && (builder= func->builder)) > return builder; > > - if ((builder= Plugin_find_native_func_builder_param(*name).find(thd))) > + if ((builder= function_plugin_find_native_function_builder(thd, *name))) this is what I mean by "special code path for plugins" and want to get rid of. But not in this commit, I agree. > return builder; > > -#ifdef HAVE_SPATIAL > - if (!builder) > - builder= plugin_function_collection_geometry. > - find_native_function_builder(thd, *name); > -#endif > - > - return builder; > + return NULL; > } > > Create_qfunc * Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] c776ed4d6ec: MDEV-19848 Server crashes in check_vcol_forward_refs upon INSERT DELAYED into table with long blob key
by Sergei Golubchik 14 Oct '19

14 Oct '19

Hi, Sachin! On Aug 20, Sachin Setiya wrote: > revision-id: c776ed4d6ec (mariadb-10.4.5-153-gc776ed4d6ec) > parent(s): 4ca016237f1 > author: Sachin <sachin.setiya(a)mariadb.com> > committer: Sachin <sachin.setiya(a)mariadb.com> > timestamp: 2019-07-30 03:42:21 +0530 > message: > > MDEV-19848 Server crashes in check_vcol_forward_refs upon INSERT DELAYED into table with long blob key > > There are 2 issues > > 1st:- in make_new_field when we & into new field flag we forget > LONG_UNIQUE_HASH_FIELD Flag. > > 2nd:- We are calling parse_vcol_defs on keyinfo , but they are not in right > form. We should call setup_keyinfo_hash_all before calling parse_vcol_defs This looks quite suspicious. Ideally, Delayed_insert::get_local_table() should just create a copy of the table, but not modify the original table. In your case it constantly modifies KEY's of the original table. It's kind of dirty. As for the real bug, see what parse_vcol_defs() is doing for long unique fields. In particular, it sets keypart->field->vcol_info= table->field[keypart->field->field_index]->vcol_info; but the keypart here is in the original Delayed_insert::table, while field and vcol_info is in a copy (created in Delayed_insert::get_local_table). This seems to be just wrong. I'd really prefer Delayed_insert::get_local_table to not change anything in the Delayed_insert::table. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 1

[Maria-developers] Performance of index intersection
by David Sickmiller 13 Oct '19

13 Oct '19

Hi! I've been using MySQL/MariaDB for two decades but have more recently been working with Elasticsearch. I knew to expect an inverted index to speed up querying full text fields, but I've been surprised (and a bit annoyed) at how fast ES can query structured data. (In my case, I'm largely looking for intersections of a number of varchar fields with lowish cardinality, e.g. WHERE country = 'US' AND client = 'Microsoft' AND status = 'Completed'.) Elasticsearch seems to have several things going on, but I think a core aspect, to use RDMS terminology, is that each column is indexed, and index unions/intersections are used if the WHERE clause references multiple columns. I've heard that MySQL/MariaDB has the ability to merge indexes, but I've rarely observed it in person. Googling for it yields a bunch of StackOverflow posts complaining how slow it is, with responses agreeing and explaining how to disable it. If I'm reading the MySQL/MariaDB code correctly, it looks like MariaDB will intersect indexes by looping through each index, reading the rowids of all matching keys and then, at the end (or once the buffer is full), checking whether each rowid is present in each index. I wonder if there's an opportunity to speed this up. If we first read in the rowids from one index (ideally the one with the fewest matches), we could tell the storage engine that, when reading the next index, it should skip over rowids lower than the next candidate intersection. In the best case scenario, I think this could enable InnoDB to use its page directory to skip past some of the keys, improving the performance from O(n) to O(log n). That said, this is all new to me. Maybe there's an obvious reason this wouldn't make much of an improvement, or maybe I've overlooked that it's already been done. However, if it looks promising to you folk, and it's something you'd consider merging, I'd be willing to attempt writing a PR for it. Thank you, David Sickmiller

3 6

Re: [Maria-developers] Missing memory barrier in parallel replication error handler in wait_for_prior_commit()?
by Kristian Nielsen 11 Oct '19

11 Oct '19

sujatha <sujatha.sivakumar(a)mariadb.com> writes: > I have a doubt. A simple fix as per the earlier mail discussion would > be to swap the > > order of assignments as shown below. > > wakeup_error= true > waitee= NULL > Why cannot we use the simpler approach. Please provide your inputs. This is because of the need for memory barriers, to prevent compiler and/or modern high-performance CPUs from re-ordering the memory accesses. There is nothing in C that requires the compiler to make the two assignments in the same order as in the source code. And even if it does, the CPU is free to perform the writes and/or the corresponding loads in the opposite order (the x86 architecture doesn't do that, but other architectures do). There are many resources on the net on the need for memory barriers, for example this from one of your collegues: https://mariadb.org/wp-content/uploads/2017/11/2017-11-Memory-barriers.pdf But it will be difficult to make a test case, I think. Because after correcting the order of the stores there is nowhere to put a debug-sleep, and there also is no easy way to force the CPU to take the incorrect memory order. - Kristian.

1 0

Re: [Maria-developers] 4ca016237f1: MDEV-20001 Potential dangerous regression: INSERT INTO >=100 rows fail for myisam table with HASH indexes
by Sergei Golubchik 09 Oct '19

09 Oct '19

Hi, Sachin! On Aug 20, Sachin Setiya wrote: > revision-id: 4ca016237f1 (mariadb-10.4.5-152-g4ca016237f1) > parent(s): 4a5cd407289 > author: Sachin <sachin.setiya(a)mariadb.com> > committer: Sachin <sachin.setiya(a)mariadb.com> > timestamp: 2019-07-29 19:33:05 +0530 > message: > > MDEV-20001 Potential dangerous regression: INSERT INTO >=100 rows fail for myisam table with HASH indexes > > Dont deactivate the long unique keys on bulk insert. > > diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc > index f478e01e441..c1169737911 100644 > --- a/storage/myisam/ha_myisam.cc > +++ b/storage/myisam/ha_myisam.cc > @@ -1749,7 +1749,16 @@ void ha_myisam::start_bulk_insert(ha_rows rows, uint flags) > else > { > my_bool all_keys= MY_TEST(flags & HA_CREATE_UNIQUE_INDEX_BY_SORT); > - mi_disable_indexes_for_rebuild(file, rows, all_keys); > + if (table->s->long_unique_table) > + { > + ulonglong hash_key_map= 0ULL; > + for(uint i= 0; i < table->s->keys; i++) > + if (table->key_info[i].algorithm == HA_KEY_ALG_LONG_HASH) > + mi_set_key_active(hash_key_map, i); > + mi_disable_indexes_for_rebuild(file, rows, all_keys, hash_key_map); > + } > + else > + mi_disable_indexes_for_rebuild(file, rows, all_keys, 0ULL); I agree with the fix, but here's a comment about the implementation. mi_disable_indexes_for_rebuild() is _only_ used here, nowhere else. So, I think I'd just removed mi_disable_indexes_for_rebuild(), inlined it here and do everything in one loop. Note, that you also need to fix Aria engine, it has almost the same code. And add a test for it, please. > } > } Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 1

Re: [Maria-developers] [Commits] d48294d15c4: MDEV-13694: Wrong result upon GROUP BY with orderby_uses_equalities=on
by Sergey Petrunia 09 Oct '19

09 Oct '19

Hi Varun, This patch is a step in the right direction, but I think we're not quite there yet. The patch adds logic about semi-join materialization into the filesort code This makes things really messy. I think, some isolation would be beneficial. The first suggestion: In Sort_param (as this structure is visible in find_all_keys), add members that control the behavior inside find_all_keys: - unpack_function (default is NULL) - bool set_all_read_bits (default is FALSE) Now, Sort_param is allocated in filesort(), so we will have to add the same members in "class Filesort" and have filesort() copy them to Sort_param (necessary because find_all_keys() doesn't see the "Filesort" object). Then, these members can be set in the SQL layer e.g. in JOIN::add_sorting_to_table which creates the Filesort structure. (It would be ideal if semi-join code set them, but this is probably not possible to achieve as Filesort is set up after the semi-join code is set up). If after all these changes "JOIN_TAB::unpacking_to_base_table_fields()" is still there, please rename it to "unpack_..." so that it follows the established convention. That's all the input. On Sat, Sep 28, 2019 at 02:12:01PM +0530, Varun wrote: > revision-id: d48294d15c4895215d5facef97fc80c03cd6b4b0 (mariadb-10.4.4-341-gd48294d15c4) > parent(s): a340af922361e3958e5d6653c8b840771db282f2 > author: Varun Gupta > committer: Varun Gupta > timestamp: 2019-09-28 13:06:44 +0530 > message: > > MDEV-13694: Wrong result upon GROUP BY with orderby_uses_equalities=on > > For the case when the SJM scan table is the first table in the join order, > then if we want to do the sorting on the SJM scan table, then we need to > make sure that we unpack the values to base table fields in two cases: > 1) Reading the SJM table and writing the sort-keys inside the sort-buffer > 2) Reading the sorted data from the sort file > > --- > mysql-test/main/order_by.result | 138 +++++++++++++++++++++++++++++++++++++++- > mysql-test/main/order_by.test | 34 ++++++++++ > sql/filesort.cc | 17 +++++ > sql/opt_subselect.cc | 10 ++- > sql/records.cc | 13 ++++ > sql/records.h | 1 + > sql/sql_select.cc | 89 ++++++++++---------------- > sql/sql_select.h | 4 +- > sql/table.h | 1 + > 9 files changed, 246 insertions(+), 61 deletions(-) > > diff --git a/mysql-test/main/order_by.result b/mysql-test/main/order_by.result > index b059cc686cd..e74583670fc 100644 > --- a/mysql-test/main/order_by.result > +++ b/mysql-test/main/order_by.result > @@ -3322,7 +3322,7 @@ WHERE books.library_id = 8663 AND > books.scheduled_for_removal=0 ) > ORDER BY wings.id; > id select_type table type possible_keys key key_len ref rows filtered Extra > -1 PRIMARY <subquery2> ALL distinct_key NULL NULL NULL 2 100.00 Using temporary; Using filesort > +1 PRIMARY <subquery2> ALL distinct_key NULL NULL NULL 2 100.00 Using filesort > 1 PRIMARY wings eq_ref PRIMARY PRIMARY 4 test.books.wings_id 1 100.00 > 2 MATERIALIZED books ref library_idx library_idx 4 const 2 100.00 Using where > Warnings: > @@ -3436,3 +3436,139 @@ Note 1003 select `test`.`t4`.`a` AS `a`,`test`.`t4`.`b` AS `b`,`test`.`t4`.`c` A > set histogram_size=@tmp_h, histogram_type=@tmp_ht, use_stat_tables=@tmp_u, > optimizer_use_condition_selectivity=@tmp_o; > drop table t1,t2,t3,t4; > +# > +# MDEV-13694: Wrong result upon GROUP BY with orderby_uses_equalities=on > +# > +CREATE TABLE t1 (a INT, b int, primary key(a)); > +CREATE TABLE t2 (a INT, b INT); > +INSERT INTO t1 (a,b) VALUES (58,1),(96,2),(273,3),(23,4),(231,5),(525,6), > +(2354,7),(321421,3),(535,2),(4535,3); > +INSERT INTO t2 (a,b) VALUES (58,3),(96,3),(273,3); > +# Join order should have the SJM scan table as the first table for both > +# the queries with GROUP BY and ORDER BY clause. > +EXPLAIN SELECT t1.a > +FROM t1 > +WHERE t1.a IN (SELECT a FROM t2 WHERE b=3) > +ORDER BY t1.a DESC; > +id select_type table type possible_keys key key_len ref rows Extra > +1 PRIMARY <subquery2> ALL distinct_key NULL NULL NULL 3 Using filesort > +1 PRIMARY t1 eq_ref PRIMARY PRIMARY 4 test.t2.a 1 Using index > +2 MATERIALIZED t2 ALL NULL NULL NULL NULL 3 Using where > +EXPLAIN FORMAT=JSON SELECT t1.a > +FROM t1 > +WHERE t1.a IN (SELECT a FROM t2 WHERE b=3) > +ORDER BY t1.a DESC; > +EXPLAIN > +{ > + "query_block": { > + "select_id": 1, > + "read_sorted_file": { > + "filesort": { > + "sort_key": "t1.a desc", > + "table": { > + "table_name": "<subquery2>", > + "access_type": "ALL", > + "possible_keys": ["distinct_key"], > + "rows": 3, > + "filtered": 100, > + "materialized": { > + "unique": 1, > + "query_block": { > + "select_id": 2, > + "table": { > + "table_name": "t2", > + "access_type": "ALL", > + "rows": 3, > + "filtered": 100, > + "attached_condition": "t2.b = 3 and t2.a is not null" > + } > + } > + } > + } > + } > + }, > + "table": { > + "table_name": "t1", > + "access_type": "eq_ref", > + "possible_keys": ["PRIMARY"], > + "key": "PRIMARY", > + "key_length": "4", > + "used_key_parts": ["a"], > + "ref": ["test.t2.a"], > + "rows": 1, > + "filtered": 100, > + "using_index": true > + } > + } > +} > +SELECT t1.a > +FROM t1 > +WHERE t1.a IN (SELECT a FROM t2 WHERE b=3) > +ORDER BY t1.a DESC; > +a > +273 > +96 > +58 > +EXPLAIN SELECT t1.a, group_concat(t1.b) > +FROM t1 > +WHERE t1.a IN (SELECT a FROM t2 WHERE b=3) > +GROUP BY t1.a DESC; > +id select_type table type possible_keys key key_len ref rows Extra > +1 PRIMARY <subquery2> ALL distinct_key NULL NULL NULL 3 Using filesort > +1 PRIMARY t1 eq_ref PRIMARY PRIMARY 4 test.t2.a 1 > +2 MATERIALIZED t2 ALL NULL NULL NULL NULL 3 Using where > +EXPLAIN FORMAT=JSON SELECT t1.a, group_concat(t1.b) > +FROM t1 > +WHERE t1.a IN (SELECT a FROM t2 WHERE b=3) > +GROUP BY t1.a DESC; > +EXPLAIN > +{ > + "query_block": { > + "select_id": 1, > + "read_sorted_file": { > + "filesort": { > + "sort_key": "t1.a desc", > + "table": { > + "table_name": "<subquery2>", > + "access_type": "ALL", > + "possible_keys": ["distinct_key"], > + "rows": 3, > + "filtered": 100, > + "materialized": { > + "unique": 1, > + "query_block": { > + "select_id": 2, > + "table": { > + "table_name": "t2", > + "access_type": "ALL", > + "rows": 3, > + "filtered": 100, > + "attached_condition": "t2.b = 3 and t2.a is not null" > + } > + } > + } > + } > + } > + }, > + "table": { > + "table_name": "t1", > + "access_type": "eq_ref", > + "possible_keys": ["PRIMARY"], > + "key": "PRIMARY", > + "key_length": "4", > + "used_key_parts": ["a"], > + "ref": ["test.t2.a"], > + "rows": 1, > + "filtered": 100 > + } > + } > +} > +SELECT t1.a, group_concat(t1.b) > +FROM t1 > +WHERE t1.a IN (SELECT a FROM t2 WHERE b=3) > +GROUP BY t1.a DESC; > +a group_concat(t1.b) > +273 3 > +96 2 > +58 1 > +DROP TABLE t1, t2; > diff --git a/mysql-test/main/order_by.test b/mysql-test/main/order_by.test > index 934c503302f..b3e43d27e2f 100644 > --- a/mysql-test/main/order_by.test > +++ b/mysql-test/main/order_by.test > @@ -2276,3 +2276,37 @@ set histogram_size=@tmp_h, histogram_type=@tmp_ht, use_stat_tables=@tmp_u, > optimizer_use_condition_selectivity=@tmp_o; > > drop table t1,t2,t3,t4; > + > + > +--echo # > +--echo # MDEV-13694: Wrong result upon GROUP BY with orderby_uses_equalities=on > +--echo # > + > +CREATE TABLE t1 (a INT, b int, primary key(a)); > +CREATE TABLE t2 (a INT, b INT); > + > +INSERT INTO t1 (a,b) VALUES (58,1),(96,2),(273,3),(23,4),(231,5),(525,6), > + (2354,7),(321421,3),(535,2),(4535,3); > +INSERT INTO t2 (a,b) VALUES (58,3),(96,3),(273,3); > + > +--echo # Join order should have the SJM scan table as the first table for both > +--echo # the queries with GROUP BY and ORDER BY clause. > + > +let $query= SELECT t1.a > + FROM t1 > + WHERE t1.a IN (SELECT a FROM t2 WHERE b=3) > + ORDER BY t1.a DESC; > + > +eval EXPLAIN $query; > +eval EXPLAIN FORMAT=JSON $query; > +eval $query; > + > +let $query= SELECT t1.a, group_concat(t1.b) > + FROM t1 > + WHERE t1.a IN (SELECT a FROM t2 WHERE b=3) > + GROUP BY t1.a DESC; > + > +eval EXPLAIN $query; > +eval EXPLAIN FORMAT=JSON $query; > +eval $query; > +DROP TABLE t1, t2; > diff --git a/sql/filesort.cc b/sql/filesort.cc > index 3f4291cfb1f..e5c83293e9f 100644 > --- a/sql/filesort.cc > +++ b/sql/filesort.cc > @@ -716,11 +716,21 @@ static ha_rows find_all_keys(THD *thd, Sort_param *param, SQL_SELECT *select, > *found_rows= 0; > ref_pos= &file->ref[0]; > next_pos=ref_pos; > + JOIN_TAB *tab= sort_form->reginfo.join_tab; > + JOIN *join= tab ? tab->join : NULL; > + bool first_is_in_sjm_nest= FALSE; > > DBUG_EXECUTE_IF("show_explain_in_find_all_keys", > dbug_serve_apcs(thd, 1); > ); > > + if (join && join->table_count != join->const_tables && > + (join->join_tab + join->const_tables == tab)) > + { > + TABLE_LIST *tbl_for_first= sort_form->pos_in_table_list; > + first_is_in_sjm_nest= tbl_for_first && tbl_for_first->is_sjm_scan_table(); > + } > + > if (!quick_select) > { > next_pos=(uchar*) 0; /* Find records in sequence */ > @@ -756,13 +766,20 @@ static ha_rows find_all_keys(THD *thd, Sort_param *param, SQL_SELECT *select, > goto err; > } > > + if (first_is_in_sjm_nest) > + sort_form->column_bitmaps_set(save_read_set, save_write_set); > + > DEBUG_SYNC(thd, "after_index_merge_phase1"); > for (;;) > { > if (quick_select) > error= select->quick->get_next(); > else /* Not quick-select */ > + { > error= file->ha_rnd_next(sort_form->record[0]); > + if (first_is_in_sjm_nest) > + tab->unpacking_to_base_table_fields(); > + } > if (unlikely(error)) > break; > file->position(sort_form->record[0]); > diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc > index 87458357865..f837a6394af 100644 > --- a/sql/opt_subselect.cc > +++ b/sql/opt_subselect.cc > @@ -4252,11 +4252,11 @@ bool setup_sj_materialization_part2(JOIN_TAB *sjm_tab) > sjm_tab->type= JT_ALL; > > /* Initialize full scan */ > - sjm_tab->read_first_record= join_read_record_no_init; > + sjm_tab->read_first_record= join_init_read_record; > sjm_tab->read_record.copy_field= sjm->copy_field; > sjm_tab->read_record.copy_field_end= sjm->copy_field + > sjm->sjm_table_cols.elements; > - sjm_tab->read_record.read_record_func= rr_sequential_and_unpack; > + sjm_tab->read_record.read_record_func= read_record_func_for_rr_and_unpack; > } > > sjm_tab->bush_children->end[-1].next_select= end_sj_materialize; > @@ -7105,3 +7105,9 @@ bool Item_in_subselect::pushdown_cond_for_in_subquery(THD *thd, Item *cond) > thd->lex->current_select= save_curr_select; > DBUG_RETURN(FALSE); > } > + > + > +bool TABLE_LIST::is_sjm_scan_table() > +{ > + return is_active_sjm() && sj_mat_info->is_sj_scan; > +} > diff --git a/sql/records.cc b/sql/records.cc > index 3d709182a4e..f6885f773d5 100644 > --- a/sql/records.cc > +++ b/sql/records.cc > @@ -709,3 +709,16 @@ static int rr_cmp(uchar *a,uchar *b) > return (int) a[7] - (int) b[7]; > #endif > } > + > + > +int read_record_func_for_rr_and_unpack(READ_RECORD *info) > +{ > + int error; > + if ((error= info->read_record_func_and_unpack_calls(info))) > + return error; > + > + for (Copy_field *cp= info->copy_field; cp != info->copy_field_end; cp++) > + (*cp->do_copy)(cp); > + > + return error; > +} > diff --git a/sql/records.h b/sql/records.h > index faf0d13c9a9..037a06b9d34 100644 > --- a/sql/records.h > +++ b/sql/records.h > @@ -55,6 +55,7 @@ struct READ_RECORD > TABLE *table; /* Head-form */ > Unlock_row_func unlock_row; > Read_func read_record_func; > + Read_func read_record_func_and_unpack_calls; > THD *thd; > SQL_SELECT *select; > uint ref_length, reclength, rec_cache_size, error_offset; > diff --git a/sql/sql_select.cc b/sql/sql_select.cc > index 36d9eda3383..28bc57c692f 100644 > --- a/sql/sql_select.cc > +++ b/sql/sql_select.cc > @@ -14015,37 +14015,8 @@ remove_const(JOIN *join,ORDER *first_order, COND *cond, > can be used without tmp. table. > */ > bool can_subst_to_first_table= false; > - bool first_is_in_sjm_nest= false; > - if (first_is_base_table) > - { > - TABLE_LIST *tbl_for_first= > - join->join_tab[join->const_tables].table->pos_in_table_list; > - first_is_in_sjm_nest= tbl_for_first->sj_mat_info && > - tbl_for_first->sj_mat_info->is_used; > - } > - /* > - Currently we do not employ the optimization that uses multiple > - equalities for ORDER BY to remove tmp table in the case when > - the first table happens to be the result of materialization of > - a semi-join nest ( <=> first_is_in_sjm_nest == true). > - > - When a semi-join nest is materialized and scanned to look for > - possible matches in the remaining tables for every its row > - the fields from the result of materialization are copied > - into the record buffers of tables from the semi-join nest. > - So these copies are used to access the remaining tables rather > - than the fields from the result of materialization. > - > - Unfortunately now this so-called 'copy back' technique is > - supported only if the rows are scanned with the rr_sequential > - function, but not with other rr_* functions that are employed > - when the result of materialization is required to be sorted. > - > - TODO: either to support 'copy back' technique for the above case, > - or to get rid of this technique altogether. > - */ > if (optimizer_flag(join->thd, OPTIMIZER_SWITCH_ORDERBY_EQ_PROP) && > - first_is_base_table && !first_is_in_sjm_nest && > + first_is_base_table && > order->item[0]->real_item()->type() == Item::FIELD_ITEM && > join->cond_equal) > { > @@ -19922,19 +19893,6 @@ do_select(JOIN *join, Procedure *procedure) > } > > > -int rr_sequential_and_unpack(READ_RECORD *info) > -{ > - int error; > - if (unlikely((error= rr_sequential(info)))) > - return error; > - > - for (Copy_field *cp= info->copy_field; cp != info->copy_field_end; cp++) > - (*cp->do_copy)(cp); > - > - return error; > -} > - > - > /** > @brief > Instantiates temporary table > @@ -21223,6 +21181,8 @@ bool test_if_use_dynamic_range_scan(JOIN_TAB *join_tab) > > int join_init_read_record(JOIN_TAB *tab) > { > + bool need_unpacking= FALSE; > + JOIN *join= tab->join; > /* > Note: the query plan tree for the below operations is constructed in > save_agg_explain_data. > @@ -21232,6 +21192,12 @@ int join_init_read_record(JOIN_TAB *tab) > if (tab->filesort && tab->sort_table()) // Sort table. > return 1; > > + if (join->top_join_tab_count != join->const_tables) > + { > + TABLE_LIST *tbl= tab->table->pos_in_table_list; > + need_unpacking= tbl ? tbl->is_sjm_scan_table() : FALSE; > + } > + > tab->build_range_rowid_filter_if_needed(); > > DBUG_EXECUTE_IF("kill_join_init_read_record", > @@ -21249,16 +21215,6 @@ int join_init_read_record(JOIN_TAB *tab) > if (!tab->preread_init_done && tab->preread_init()) > return 1; > > - > - if (init_read_record(&tab->read_record, tab->join->thd, tab->table, > - tab->select, tab->filesort_result, 1,1, FALSE)) > - return 1; > - return tab->read_record.read_record(); > -} > - > -int > -join_read_record_no_init(JOIN_TAB *tab) > -{ > Copy_field *save_copy, *save_copy_end; > > /* > @@ -21268,12 +21224,20 @@ join_read_record_no_init(JOIN_TAB *tab) > save_copy= tab->read_record.copy_field; > save_copy_end= tab->read_record.copy_field_end; > > - init_read_record(&tab->read_record, tab->join->thd, tab->table, > - tab->select, tab->filesort_result, 1, 1, FALSE); > + > + if (init_read_record(&tab->read_record, tab->join->thd, tab->table, > + tab->select, tab->filesort_result, 1, 1, FALSE)) > + return 1; > > tab->read_record.copy_field= save_copy; > tab->read_record.copy_field_end= save_copy_end; > - tab->read_record.read_record_func= rr_sequential_and_unpack; > + > + if (need_unpacking) > + { > + tab->read_record.read_record_func_and_unpack_calls= > + tab->read_record.read_record_func; > + tab->read_record.read_record_func = read_record_func_for_rr_and_unpack; > + } > > return tab->read_record.read_record(); > } > @@ -28981,6 +28945,19 @@ void build_notnull_conds_for_inner_nest_of_outer_join(JOIN *join, > } > > > +/* > + @brief > + Unpacking temp table fields to base table fields. > +*/ > + > +void JOIN_TAB::unpacking_to_base_table_fields() > +{ > + for (Copy_field *cp= read_record.copy_field; > + cp != read_record.copy_field_end; cp++) > + (*cp->do_copy)(cp); > +} > + > + > /** > @} (end of group Query_Optimizer) > */ > diff --git a/sql/sql_select.h b/sql/sql_select.h > index 4f7bf49f635..545d4a788cc 100644 > --- a/sql/sql_select.h > +++ b/sql/sql_select.h > @@ -223,7 +223,7 @@ typedef enum_nested_loop_state > (*Next_select_func)(JOIN *, struct st_join_table *, bool); > Next_select_func setup_end_select_func(JOIN *join, JOIN_TAB *tab); > int rr_sequential(READ_RECORD *info); > -int rr_sequential_and_unpack(READ_RECORD *info); > +int read_record_func_for_rr_and_unpack(READ_RECORD *info); > Item *remove_pushed_top_conjuncts(THD *thd, Item *cond); > Item *and_new_conditions_to_optimized_cond(THD *thd, Item *cond, > COND_EQUAL **cond_eq, > @@ -676,6 +676,7 @@ typedef struct st_join_table { > table_map remaining_tables); > bool fix_splitting(SplM_plan_info *spl_plan, table_map remaining_tables, > bool is_const_table); > + void unpacking_to_base_table_fields(); > } JOIN_TAB; > > > @@ -2352,7 +2353,6 @@ create_virtual_tmp_table(THD *thd, Field *field) > > int test_if_item_cache_changed(List<Cached_item> &list); > int join_init_read_record(JOIN_TAB *tab); > -int join_read_record_no_init(JOIN_TAB *tab); > void set_position(JOIN *join,uint idx,JOIN_TAB *table,KEYUSE *key); > inline Item * and_items(THD *thd, Item* cond, Item *item) > { > diff --git a/sql/table.h b/sql/table.h > index 1a7e5fbd4dc..35ba9bbb95d 100644 > --- a/sql/table.h > +++ b/sql/table.h > @@ -2622,6 +2622,7 @@ struct TABLE_LIST > */ > const char *get_table_name() const { return view != NULL ? view_name.str : table_name.str; } > bool is_active_sjm(); > + bool is_sjm_scan_table(); > bool is_jtbm() { return MY_TEST(jtbm_subselect != NULL); } > st_select_lex_unit *get_unit(); > st_select_lex *get_single_select(); > _______________________________________________ > commits mailing list > commits(a)mariadb.org > https://lists.askmonty.org/cgi-bin/mailman/listinfo/commits -- BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

[Maria-developers] GSoC: On completing the MDEV-6017 (Add support for Indexes on Expressions)
by Alexey Mogilyovkin 01 Oct '19

01 Oct '19

Hello, I would like to finish the task I was working on during GSoC. Nikita told me that there are some architectural questions to my code. Can you tell about them in more detail?

2 1

Re: [Maria-developers] 3f38da9145c: MDEV-20016 Add MariaDB_DATA_TYPE_PLUGIN
by Sergei Golubchik 27 Sep '19

27 Sep '19

Hi, Alexander! On Sep 16, Alexander Barkov wrote: > revision-id: 3f38da9145c (mariadb-10.4.4-251-g3f38da9145c) > parent(s): e6ff3f9d1c8 > author: Alexander Barkov <bar(a)mariadb.com> > committer: Alexander Barkov <bar(a)mariadb.com> > timestamp: 2019-07-12 07:53:55 +0400 > message: > > MDEV-20016 Add MariaDB_DATA_TYPE_PLUGIN > diff --git a/include/mysql/plugin.h b/include/mysql/plugin.h > index 85e52a247af..92703b626ac 100644 > --- a/include/mysql/plugin.h > +++ b/include/mysql/plugin.h > @@ -611,6 +612,22 @@ struct handlerton; > int interface_version; > }; > > +/* > + API for data type plugin. (MariaDB_DATA_TYPE_PLUGIN) > +*/ > +#define MariaDB_DATA_TYPE_INTERFACE_VERSION (MYSQL_VERSION_ID << 8) > + > +/** > + Data type plugin descriptor > +*/ > +#ifdef __cplusplus > +struct st_mariadb_data_type > +{ > + int interface_version; > + const class Type_handler *type_handler; > +}; > +#endif Plugin-wise it'd be better to have a separate plugin_data_type.h and put Type_handler definition there. So that as much of the API as possible gets into the .pp file and we could detect when it changes. > + > /************************************************************************* > st_mysql_value struct for reading values from mysqld. > Used by server variables framework to parse user-provided values. > diff --git a/plugin/type_test/CMakeLists.txt b/plugin/type_test/CMakeLists.txt > new file mode 100644 > index 00000000000..b85168d1bd2 > --- /dev/null > +++ b/plugin/type_test/CMakeLists.txt > @@ -0,0 +1,17 @@ > +# Copyright (c) 2016, MariaDB corporation. All rights reserved. > +# > +# This program is free software; you can redistribute it and/or modify > +# it under the terms of the GNU General Public License as published by > +# the Free Software Foundation; version 2 of the License. > +# > +# This program is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +# GNU General Public License for more details. > +# > +# You should have received a copy of the GNU General Public License > +# along with this program; if not, write to the Free Software > +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA > + > +MYSQL_ADD_PLUGIN(type_test plugin.cc RECOMPILE_FOR_EMBEDDED > + MODULE_ONLY COMPONENT Test) Add at least two new plugins, please. May be in separate commits, as you like. But at least two. And practical, not just to test the API > diff --git a/plugin/type_test/mysql-test/type_test/type_test_int8-debug.test b/plugin/type_test/mysql-test/type_test/type_test_int8-debug.test > new file mode 100644 > index 00000000000..30e313481d6 > --- /dev/null > +++ b/plugin/type_test/mysql-test/type_test/type_test_int8-debug.test > @@ -0,0 +1,11 @@ > +--echo # > +--echo # MDEV-20016 Add MariaDB_DATA_TYPE_PLUGIN > +--echo # > + > +SET SESSION debug_dbug="+d,frm_data_type_info"; > + > +CREATE TABLE t1 (a TEST_INT8); > +SHOW CREATE TABLE t1; > +DROP TABLE t1; > + > +SET SESSION debug_dbug="-d,frm_data_type_info"; don't do that, always save old debug_dbug value instead and restore it later. SET @old_debug_dbug=@@debug_dbug; SET @@debug_dbug="+d,frm_data_type_info"; ... SET @@debug_dbug=@old_debug_dbug; because after "+d,xxx" you have one keyword enabled, like in @@debug_dbug="d,xxx". and after "-d,xxx" you have no keywords enabled, like in @debug_dbug="d" which means "all keywords enabled" for dbug. > diff --git a/plugin/type_test/mysql-test/type_test/type_test_int8.result b/plugin/type_test/mysql-test/type_test/type_test_int8.result > new file mode 100644 > index 00000000000..758f94904c1 > --- /dev/null > +++ b/plugin/type_test/mysql-test/type_test/type_test_int8.result > @@ -0,0 +1,144 @@ > +# > +# MDEV-20016 Add MariaDB_DATA_TYPE_PLUGIN > +# > +SELECT > +PLUGIN_NAME, > +PLUGIN_VERSION, > +PLUGIN_STATUS, > +PLUGIN_TYPE, > +PLUGIN_AUTHOR, > +PLUGIN_DESCRIPTION, > +PLUGIN_LICENSE, > +PLUGIN_MATURITY, > +PLUGIN_AUTH_VERSION > +FROM INFORMATION_SCHEMA.PLUGINS > +WHERE PLUGIN_TYPE='DATA TYPE' > + AND PLUGIN_NAME='TEST_INT8'; > +PLUGIN_NAME TEST_INT8 > +PLUGIN_VERSION 1.0 > +PLUGIN_STATUS ACTIVE > +PLUGIN_TYPE DATA TYPE > +PLUGIN_AUTHOR MariaDB > +PLUGIN_DESCRIPTION Data type TEST_INT8 > +PLUGIN_LICENSE GPL > +PLUGIN_MATURITY Alpha > +PLUGIN_AUTH_VERSION 1.0 > +CREATE TABLE t1 (a TEST_INT8); > +SHOW CREATE TABLE t1; > +Table Create Table > +t1 CREATE TABLE `t1` ( > + `a` test_int8 DEFAULT NULL is the type name a reserved word? or an arbitrary identifier? should we support (and print as) `a` `test_int8` DEFAULT NULL ? > +) ENGINE=MyISAM DEFAULT CHARSET=latin1 > +DROP TABLE t1; > +SELECT CAST('100' AS TEST_INT8) AS cast; > +cast > +100 > +BEGIN NOT ATOMIC > +DECLARE a TEST_INT8 DEFAULT 256; > +SELECT HEX(a), a; > +END; > +$$ > +HEX(a) a > +100 256 > +CREATE FUNCTION f1(p TEST_INT8) RETURNS TEST_INT8 RETURN 1; > +SHOW CREATE FUNCTION f1; > +Function sql_mode Create Function character_set_client collation_connection Database Collation > +f1 STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION CREATE DEFINER=`root`@`localhost` FUNCTION `f1`(p TEST_INT8) RETURNS test_int8 > +RETURN 1 latin1 latin1_swedish_ci latin1_swedish_ci use enable_metadata for a part (or for a whole) of this test and show how mysql --column-type-info handles it. > +SELECT f1(10); > +f1(10) > +1 > +DROP FUNCTION f1; > +CREATE TABLE t1 (a TEST_INT8); > +CREATE TABLE t2 AS SELECT a FROM t1; > +SHOW CREATE TABLE t2; > +Table Create Table > +t2 CREATE TABLE `t2` ( > + `a` test_int8 DEFAULT NULL > +) ENGINE=MyISAM DEFAULT CHARSET=latin1 > +DROP TABLE t2; > +DROP TABLE t1; > +CREATE TABLE t1 (a TEST_INT8); > +CREATE TABLE t2 AS SELECT COALESCE(a,a) FROM t1; > +SHOW CREATE TABLE t2; > +Table Create Table > +t2 CREATE TABLE `t2` ( > + `COALESCE(a,a)` test_int8 DEFAULT NULL > +) ENGINE=MyISAM DEFAULT CHARSET=latin1 > +DROP TABLE t2; > +DROP TABLE t1; > +CREATE TABLE t1 (a TEST_INT8); > +CREATE TABLE t2 AS SELECT LEAST(a,a) FROM t1; > +SHOW CREATE TABLE t2; > +Table Create Table > +t2 CREATE TABLE `t2` ( > + `LEAST(a,a)` test_int8 DEFAULT NULL > +) ENGINE=MyISAM DEFAULT CHARSET=latin1 > +DROP TABLE t2; > +DROP TABLE t1; > +CREATE TABLE t1 (a TEST_INT8); > +INSERT INTO t1 VALUES (1),(2); > +CREATE TABLE t2 AS SELECT MIN(a), MAX(a) FROM t1; > +SELECT * FROM t2; > +MIN(a) MAX(a) > +1 2 > +SHOW CREATE TABLE t2; > +Table Create Table > +t2 CREATE TABLE `t2` ( > + `MIN(a)` test_int8 DEFAULT NULL, > + `MAX(a)` test_int8 DEFAULT NULL > +) ENGINE=MyISAM DEFAULT CHARSET=latin1 > +DROP TABLE t2; > +DROP TABLE t1; > +CREATE TABLE t1 (id INT, a TEST_INT8); > +INSERT INTO t1 VALUES (1,1),(1,2),(2,1),(2,2); > +CREATE TABLE t2 AS SELECT id, MIN(a), MAX(a) FROM t1 GROUP BY id; > +SELECT * FROM t2; > +id MIN(a) MAX(a) > +1 1 2 > +2 1 2 > +SHOW CREATE TABLE t2; > +Table Create Table > +t2 CREATE TABLE `t2` ( > + `id` int(11) DEFAULT NULL, > + `MIN(a)` test_int8 DEFAULT NULL, > + `MAX(a)` test_int8 DEFAULT NULL > +) ENGINE=MyISAM DEFAULT CHARSET=latin1 > +DROP TABLE t2; > +DROP TABLE t1; > +CREATE TABLE t1 (a TEST_INT8); > +INSERT INTO t1 VALUES (1),(2); > +CREATE TABLE t2 AS SELECT DISTINCT a FROM t1; > +SELECT * FROM t2; > +a > +1 > +2 > +SHOW CREATE TABLE t2; > +Table Create Table > +t2 CREATE TABLE `t2` ( > + `a` test_int8 DEFAULT NULL > +) ENGINE=MyISAM DEFAULT CHARSET=latin1 > +DROP TABLE t2; > +DROP TABLE t1; > +CREATE TABLE t1 (a TEST_INT8); > +INSERT INTO t1 VALUES (1); > +CREATE TABLE t2 AS SELECT (SELECT a FROM t1) AS c1; > +SELECT * FROM t2; > +c1 > +1 > +SHOW CREATE TABLE t2; > +Table Create Table > +t2 CREATE TABLE `t2` ( > + `c1` test_int8 DEFAULT NULL > +) ENGINE=MyISAM DEFAULT CHARSET=latin1 > +DROP TABLE t2; > +DROP TABLE t1; > +CREATE TABLE t1 (a TEST_INT8); > +CREATE TABLE t2 AS SELECT a FROM t1 UNION SELECT a FROM t1; > +SHOW CREATE TABLE t2; > +Table Create Table > +t2 CREATE TABLE `t2` ( > + `a` test_int8 DEFAULT NULL > +) ENGINE=MyISAM DEFAULT CHARSET=latin1 > +DROP TABLE t2; > +DROP TABLE t1; CREATE LIKE, ALTER TABLE, SHOW COLUMNS, I_S.COLUMNS Does it support indexing? > diff --git a/plugin/type_test/plugin.cc b/plugin/type_test/plugin.cc > new file mode 100644 > index 00000000000..ea70c70f786 > --- /dev/null > +++ b/plugin/type_test/plugin.cc > @@ -0,0 +1,120 @@ > +/* > + Copyright (c) 2000, 2015, Oracle and/or its affiliates. > + Copyright (c) 2009, 2019, MariaDB > + > + This program is free software; you can redistribute it and/or modify > + it under the terms of the GNU General Public License as published by > + the Free Software Foundation; version 2 of the License. > + > + This program is distributed in the hope that it will be useful, > + but WITHOUT ANY WARRANTY; without even the implied warranty of > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > + GNU General Public License for more details. > + > + You should have received a copy of the GNU General Public License > + along with this program; if not, write to the Free Software > + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ > + > +#include <my_global.h> > +#include <sql_class.h> // THD > +#include <mysql/plugin.h> > +#include "sql_type.h" > + > + > +class Field_test_int8 :public Field_longlong > +{ > +public: > + Field_test_int8(const LEX_CSTRING &name, const Record_addr &addr, > + enum utype unireg_check_arg, > + uint32 len_arg, bool zero_arg, bool unsigned_arg) > + :Field_longlong(addr.ptr(), len_arg, addr.null_ptr(), addr.null_bit(), > + Field::NONE, &name, zero_arg, unsigned_arg) > + {} > + void sql_type(String &res) const > + { > + CHARSET_INFO *cs= res.charset(); > + res.length(cs->cset->snprintf(cs,(char*) res.ptr(),res.alloced_length(), > + "test_int8")); isn't there an easier way of setting a String to a specific value? Like, res.copy() or something? By the way, why do you need to do it at all, if the parent's Field::sql_type method could set the value from type_handler()->name() ? > + // UNSIGNED and ZEROFILL flags are not supported by the parser yet. > + // add_zerofill_and_unsigned(res); > + } > + const Type_handler *type_handler() const; > +}; > + > + > +class Type_handler_test_int8: public Type_handler_longlong > +{ > +public: > + const Name name() const override > + { > + static Name name(STRING_WITH_LEN("test_int8")); I'd prefer this being picked up automatically from the plugin name. Like it's done for engines, I_S tables, auth plugins, ft parsers, etc. > + return name; > + } > + bool Column_definition_data_type_info_image(Binary_string *to, > + const Column_definition &def) > + const override > + { > + return to->append(Type_handler_test_int8::name().lex_cstring()); > + } Not sure, why this has to be overridden by the derived class. > + Field *make_table_field(MEM_ROOT *root, > + const LEX_CSTRING *name, > + const Record_addr &addr, > + const Type_all_attributes &attr, > + TABLE *table) const override > + { > + return new (root) > + Field_test_int8(*name, addr, Field::NONE, > + attr.max_char_length(), > + 0/*zerofill*/, > + attr.unsigned_flag); > + } > + > + Field *make_table_field_from_def(TABLE_SHARE *share, MEM_ROOT *root, > + const LEX_CSTRING *name, > + const Record_addr &rec, const Bit_addr &bit, > + const Column_definition_attributes *attr, > + uint32 flags) const override > + { > + return new (root) > + Field_test_int8(*name, rec, attr->unireg_check, > + (uint32) attr->length, > + f_is_zerofill(attr->pack_flag) != 0, > + f_is_dec(attr->pack_flag) == 0); > + } > +}; Why do you need two different methods? I'd expect one that does new Field_test_int8() to be enough. The second can be in the parent class, calling make_table_field(). Or both could be in the parent class, calling the only virtual method that actually does new (root) Field_test_int8(...) > + > +static Type_handler_test_int8 type_handler_test_int8; > + > + > +const Type_handler *Field_test_int8::type_handler() const > +{ > + return &type_handler_test_int8; > +} > + > + > +/*************************************************************************/ > + > +static struct st_mariadb_data_type data_type_test_plugin= > +{ > + MariaDB_DATA_TYPE_INTERFACE_VERSION, > + &type_handler_test_int8 > +}; It's be more interesting to have a distinct type, not just an alias for BIGINT. E.g. 7-byte integer. As an example, it's a pretty empty plugin, it doesn't show why this API was ever created. Add something non-trivial to it please. And some real, non-test, plugin in a separate commit. > + > + > +maria_declare_plugin(type_geom) > +{ > + MariaDB_DATA_TYPE_PLUGIN, // the plugin type (see include/mysql/plugin.h) > + &data_type_test_plugin, // pointer to type-specific plugin descriptor > + "TEST_INT8", // plugin name > + "MariaDB", // plugin author MariaDB Corporation ? > + "Data type TEST_INT8", // the plugin description > + PLUGIN_LICENSE_GPL, // the plugin license (see include/mysql/plugin.h) > + 0, // Pointer to plugin initialization function > + 0, // Pointer to plugin deinitialization function > + 0x0100, // Numeric version 0xAABB means AA.BB veriosn > + NULL, // Status variables > + NULL, // System variables > + "1.0", // String version representation > + MariaDB_PLUGIN_MATURITY_ALPHA // Maturity (see include/mysql/plugin.h)*/ EXPERIMENTAL (for test plugins. ALPHA kind of implies it'll be BETA, GAMMA and STABLE eventually) > +} > +maria_declare_plugin_end; > diff --git a/sql/sql_type.cc b/sql/sql_type.cc > index 5cecd9f50f7..988619cb5f9 100644 > --- a/sql/sql_type.cc > +++ b/sql/sql_type.cc > @@ -121,8 +121,23 @@ bool Type_handler_data::init() > > > const Type_handler * > -Type_handler::handler_by_name(const LEX_CSTRING &name) > +Type_handler::handler_by_name(THD *thd, const LEX_CSTRING &name) > { > + plugin_ref plugin; > + if ((plugin= my_plugin_lock_by_name(thd, &name, MariaDB_DATA_TYPE_PLUGIN))) > + { > + /* > + INSTALL PLUGIN is not fully supported for data type plugins yet. Why? What's not supported? > + Fow now we have only mandatory built-in plugins > + and dynamic plugins for test purposes, > + Should be safe to unlock the plugin immediately. > + */ > + const Type_handler *ph= reinterpret_cast<st_mariadb_data_type*> > + (plugin_decl(plugin)->info)->type_handler; > + plugin_unlock(thd, plugin); > + return ph; > + } > + > #ifdef HAVE_SPATIAL > const Type_handler *ha= type_collection_geometry.handler_by_name(name); > if (ha) > diff --git a/sql/sql_type.h b/sql/sql_type.h > index 8f2d4d0c49d..b01330f30e4 100644 > --- a/sql/sql_type.h > +++ b/sql/sql_type.h > @@ -3221,9 +3221,9 @@ class Information_schema_character_attributes > class Type_handler > { > protected: > - static const Name m_version_default; > - static const Name m_version_mysql56; > - static const Name m_version_mariadb53; > + static const MYSQL_PLUGIN_IMPORT Name m_version_default; > + static const MYSQL_PLUGIN_IMPORT Name m_version_mysql56; > + static const MYSQL_PLUGIN_IMPORT Name m_version_mariadb53; Why do you need that? Parent's behavior should be always fine for plugins. I don't think plugins should know about this at all. > String *print_item_value_csstr(THD *thd, Item *item, String *str) const; > String *print_item_value_temporal(THD *thd, Item *item, String *str, > const Name &type_name, String *buf) const; > @@ -5129,9 +5130,9 @@ class Type_handler_bool: public Type_handler_long > > class Type_handler_longlong: public Type_handler_general_purpose_int > { > - static const Name m_name_longlong; > - static const Type_limits_int m_limits_sint64; > - static const Type_limits_int m_limits_uint64; > + static const MYSQL_PLUGIN_IMPORT Name m_name_longlong; > + static const MYSQL_PLUGIN_IMPORT Type_limits_int m_limits_sint64; > + static const MYSQL_PLUGIN_IMPORT Type_limits_int m_limits_uint64; same here > public: > virtual ~Type_handler_longlong() {} > const Name name() const { return m_name_longlong; } > diff --git a/sql/table.cc b/sql/table.cc > index ea333cb2ecd..70b3814df6c 100644 > --- a/sql/table.cc > +++ b/sql/table.cc > @@ -2291,12 +2291,20 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, > > if (field_data_type_info_array.count()) > { > + const LEX_CSTRING &info= field_data_type_info_array. > + element(i).type_info(); > DBUG_EXECUTE_IF("frm_data_type_info", > push_warning_printf(thd, Sql_condition::WARN_LEVEL_NOTE, > ER_UNKNOWN_ERROR, "DBUG: [%u] name='%s' type_info='%.*s'", > i, share->fieldnames.type_names[i], > - (uint) field_data_type_info_array.element(i).type_info().length, > - field_data_type_info_array.element(i).type_info().str);); > + (uint) info.length, info.str);); > + > + if (info.length) > + { > + const Type_handler *h= Type_handler::handler_by_name(thd, info); > + if (h) > + handler= h; > + } I don't see where you handle the error that "unknown type" > } > } Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 3

Re: [Maria-developers] [Commits] 673e2537249: Fix missing memory barrier in wait_for_commit
by Kristian Nielsen 26 Sep '19

26 Sep '19

Hi Andrei, > > I noticed another thing with the wait_for_commit error handling while > > looking at the MDEV-18648 patch. > So practically, we seem have to turn the types of the pair into > std::atomic<> to store into, in the right order, and load from (the way > it is). Yes. Something like this? There is a bit of noise in the patch due to changing the type of waitee to atomic and adjusting all code that accesses it. But it's basically this in wakeup: this->wakeup_error= wakeup_error; waitee.store(NULL, std::memory_order_release); and this in the waiter fast path (one place in sql_class.h and one in log.cc): if (waitee.load(std::memory_order_acquire)) return wait_for_prior_commit2(thd); else return wakeup_error; Other access are protected by lock or happen within the waiting thread only, so they can just use the memory_order_relaxed access semantics without barriers. Unfortunately I don't see how to do a useful test case. It is not really possible to force load or store reordering (on x86 I believe the CPU in fact enforces orderring between loads and stores). I couldn't think of a test that would fail without the patch and succeed with it. If you agree with the patch, do you have an opinion on which version it should go into? I'd suggest 10.4, since (IIUC) that version started the use of std::atomic. (Patch also available here: https://github.com/knielsen/server/commit/673e253724979fd9fe43a4a22bd7e1b2c…) - Kristian. Kristian Nielsen <knielsen(a)knielsen-hq.org> writes: > revision-id: 673e253724979fd9fe43a4a22bd7e1b2c3a5269e (mariadb-10.4.4-333-g673e2537249) > parent(s): 8887effe13ad87ba0460d4d3068fb5696f089bb0 > author: Kristian Nielsen > committer: Kristian Nielsen > timestamp: 2019-09-26 17:43:26 +0200 > message: > > Fix missing memory barrier in wait_for_commit > > The function wait_for_commit::wait_for_prior_commit() has a fast path > where it checks without locks if wakeup_subsequent_commits() has > already been called. This check was missing a memory barrier. The > waitee thread does two writes to variables `waitee' and > `wakeup_error', and if the waiting thread sees the first write it > _must_ also see the second or incorrect behaviour will occur. This > requires memory barriers between both the writes (release semantics) > and the reads (acquire semantics) of those two variables. > > Other accesses to these variables are done under lock or where only > one thread will be accessing them, and can be done without barriers > (relaxed sematics). > > --- > sql/log.cc | 19 +++++++++++++------ > sql/sql_class.cc | 25 ++++++++++++++----------- > sql/sql_class.h | 17 ++++++++++++----- > 3 files changed, 39 insertions(+), 22 deletions(-) > > diff --git a/sql/log.cc b/sql/log.cc > index 4f51a9a9c17..a88d5147898 100644 > --- a/sql/log.cc > +++ b/sql/log.cc > @@ -7467,8 +7467,10 @@ MYSQL_BIN_LOG::queue_for_group_commit(group_commit_entry *orig_entry) > */ > wfc= orig_entry->thd->wait_for_commit_ptr; > orig_entry->queued_by_other= false; > - if (wfc && wfc->waitee) > + if (wfc && wfc->waitee.load(std::memory_order_acquire)) > { > + wait_for_commit *loc_waitee; > + > mysql_mutex_lock(&wfc->LOCK_wait_commit); > /* > Do an extra check here, this time safely under lock. > @@ -7480,10 +7482,10 @@ MYSQL_BIN_LOG::queue_for_group_commit(group_commit_entry *orig_entry) > before setting the flag, so there is no risk that we can queue ahead of > it. > */ > - if (wfc->waitee && !wfc->waitee->commit_started) > + if ((loc_waitee= wfc->waitee.load(std::memory_order_relaxed)) && > + !loc_waitee->commit_started) > { > PSI_stage_info old_stage; > - wait_for_commit *loc_waitee; > > /* > By setting wfc->opaque_pointer to our own entry, we mark that we are > @@ -7505,7 +7507,8 @@ MYSQL_BIN_LOG::queue_for_group_commit(group_commit_entry *orig_entry) > &wfc->LOCK_wait_commit, > &stage_waiting_for_prior_transaction_to_commit, > &old_stage); > - while ((loc_waitee= wfc->waitee) && !orig_entry->thd->check_killed(1)) > + while ((loc_waitee= wfc->waitee.load(std::memory_order_relaxed)) && > + !orig_entry->thd->check_killed(1)) > mysql_cond_wait(&wfc->COND_wait_commit, &wfc->LOCK_wait_commit); > wfc->opaque_pointer= NULL; > DBUG_PRINT("info", ("After waiting for prior commit, queued_by_other=%d", > @@ -7523,14 +7526,18 @@ MYSQL_BIN_LOG::queue_for_group_commit(group_commit_entry *orig_entry) > do > { > mysql_cond_wait(&wfc->COND_wait_commit, &wfc->LOCK_wait_commit); > - } while (wfc->waitee); > + } while (wfc->waitee.load(std::memory_order_relaxed)); > } > else > { > /* We were killed, so remove us from the list of waitee. */ > wfc->remove_from_list(&loc_waitee->subsequent_commits_list); > mysql_mutex_unlock(&loc_waitee->LOCK_wait_commit); > - wfc->waitee= NULL; > + /* > + This is the thread clearing its own status, it is no longer on > + the list of waiters. So no memory barriers are needed here. > + */ > + wfc->waitee.store(NULL, std::memory_order_relaxed); > > orig_entry->thd->EXIT_COND(&old_stage); > /* Interrupted by kill. */ > diff --git a/sql/sql_class.cc b/sql/sql_class.cc > index 4eab241232b..ca179a39dc1 100644 > --- a/sql/sql_class.cc > +++ b/sql/sql_class.cc > @@ -7230,7 +7230,7 @@ wait_for_commit::reinit() > { > subsequent_commits_list= NULL; > next_subsequent_commit= NULL; > - waitee= NULL; > + waitee.store(NULL, std::memory_order_relaxed); > opaque_pointer= NULL; > wakeup_error= 0; > wakeup_subsequent_commits_running= false; > @@ -7308,8 +7308,9 @@ wait_for_commit::wakeup(int wakeup_error) > > */ > mysql_mutex_lock(&LOCK_wait_commit); > - waitee= NULL; > this->wakeup_error= wakeup_error; > + /* Memory barrier to make wakeup_error visible to the waiter thread. */ > + waitee.store(NULL, std::memory_order_release); > /* > Note that it is critical that the mysql_cond_signal() here is done while > still holding the mutex. As soon as we release the mutex, the waiter might > @@ -7340,9 +7341,10 @@ wait_for_commit::wakeup(int wakeup_error) > void > wait_for_commit::register_wait_for_prior_commit(wait_for_commit *waitee) > { > - DBUG_ASSERT(!this->waitee /* No prior registration allowed */); > + DBUG_ASSERT(!this->waitee.load(std::memory_order_relaxed) > + /* No prior registration allowed */); > wakeup_error= 0; > - this->waitee= waitee; > + this->waitee.store(waitee, std::memory_order_relaxed); > > mysql_mutex_lock(&waitee->LOCK_wait_commit); > /* > @@ -7351,7 +7353,7 @@ wait_for_commit::register_wait_for_prior_commit(wait_for_commit *waitee) > see comments on wakeup_subsequent_commits2() for details. > */ > if (waitee->wakeup_subsequent_commits_running) > - this->waitee= NULL; > + this->waitee.store(NULL, std::memory_order_relaxed); > else > { > /* > @@ -7381,7 +7383,8 @@ wait_for_commit::wait_for_prior_commit2(THD *thd) > thd->ENTER_COND(&COND_wait_commit, &LOCK_wait_commit, > &stage_waiting_for_prior_transaction_to_commit, > &old_stage); > - while ((loc_waitee= this->waitee) && likely(!thd->check_killed(1))) > + while ((loc_waitee= this->waitee.load(std::memory_order_relaxed)) && > + likely(!thd->check_killed(1))) > mysql_cond_wait(&COND_wait_commit, &LOCK_wait_commit); > if (!loc_waitee) > { > @@ -7404,14 +7407,14 @@ wait_for_commit::wait_for_prior_commit2(THD *thd) > do > { > mysql_cond_wait(&COND_wait_commit, &LOCK_wait_commit); > - } while (this->waitee); > + } while (this->waitee.load(std::memory_order_relaxed)); > if (wakeup_error) > my_error(ER_PRIOR_COMMIT_FAILED, MYF(0)); > goto end; > } > remove_from_list(&loc_waitee->subsequent_commits_list); > mysql_mutex_unlock(&loc_waitee->LOCK_wait_commit); > - this->waitee= NULL; > + this->waitee.store(NULL, std::memory_order_relaxed); > > wakeup_error= thd->killed_errno(); > if (!wakeup_error) > @@ -7513,7 +7516,7 @@ wait_for_commit::unregister_wait_for_prior_commit2() > wait_for_commit *loc_waitee; > > mysql_mutex_lock(&LOCK_wait_commit); > - if ((loc_waitee= this->waitee)) > + if ((loc_waitee= this->waitee.load(std::memory_order_relaxed))) > { > mysql_mutex_lock(&loc_waitee->LOCK_wait_commit); > if (loc_waitee->wakeup_subsequent_commits_running) > @@ -7526,7 +7529,7 @@ wait_for_commit::unregister_wait_for_prior_commit2() > See comments on wakeup_subsequent_commits2() for more details. > */ > mysql_mutex_unlock(&loc_waitee->LOCK_wait_commit); > - while (this->waitee) > + while (this->waitee.load(std::memory_order_relaxed)) > mysql_cond_wait(&COND_wait_commit, &LOCK_wait_commit); > } > else > @@ -7534,7 +7537,7 @@ wait_for_commit::unregister_wait_for_prior_commit2() > /* Remove ourselves from the list in the waitee. */ > remove_from_list(&loc_waitee->subsequent_commits_list); > mysql_mutex_unlock(&loc_waitee->LOCK_wait_commit); > - this->waitee= NULL; > + this->waitee.store(NULL, std::memory_order_relaxed); > } > } > wakeup_error= 0; > diff --git a/sql/sql_class.h b/sql/sql_class.h > index 72cb8148895..1c81739865b 100644 > --- a/sql/sql_class.h > +++ b/sql/sql_class.h > @@ -20,6 +20,7 @@ > > /* Classes in mysql */ > > +#include <atomic> > #include "dur_prop.h" > #include <waiting_threads.h> > #include "sql_const.h" > @@ -2018,7 +2019,7 @@ struct wait_for_commit > /* > The LOCK_wait_commit protects the fields subsequent_commits_list and > wakeup_subsequent_commits_running (for a waitee), and the pointer > - waiterr and associated COND_wait_commit (for a waiter). > + waitee and associated COND_wait_commit (for a waiter). > */ > mysql_mutex_t LOCK_wait_commit; > mysql_cond_t COND_wait_commit; > @@ -2032,8 +2033,14 @@ struct wait_for_commit > > When this is cleared for wakeup, the COND_wait_commit condition is > signalled. > + > + This pointer is protected by LOCK_wait_commit. But there is also a "fast > + path" where the waiter compares this to NULL without holding the lock. > + Such read must be done with acquire semantics (and all corresponding > + writes done with release semantics). This ensures that a wakeup with error > + is reliably detected as (waitee==NULL && wakeup_error != 0). > */ > - wait_for_commit *waitee; > + std::atomic<wait_for_commit *> waitee; > /* > Generic pointer for use by the transaction coordinator to optimise the > waiting for improved group commit. > @@ -2068,7 +2075,7 @@ struct wait_for_commit > Quick inline check, to avoid function call and locking in the common case > where no wakeup is registered, or a registered wait was already signalled. > */ > - if (waitee) > + if (waitee.load(std::memory_order_acquire)) > return wait_for_prior_commit2(thd); > else > { > @@ -2096,7 +2103,7 @@ struct wait_for_commit > } > void unregister_wait_for_prior_commit() > { > - if (waitee) > + if (waitee.load(std::memory_order_relaxed)) > unregister_wait_for_prior_commit2(); > else > wakeup_error= 0; > @@ -2118,7 +2125,7 @@ struct wait_for_commit > } > next_ptr_ptr= &cur->next_subsequent_commit; > } > - waitee= NULL; > + waitee.store(NULL, std::memory_order_relaxed); > } > > void wakeup(int wakeup_error); > _______________________________________________ > commits mailing list > commits(a)mariadb.org > https://lists.askmonty.org/cgi-bin/mailman/listinfo/commits

1 0

Re: [Maria-developers] [Commits] e07caf401c2: MDEV-20645: Replication consistency is broken as workers miss the error notification from an earlier failed group.
by Kristian Nielsen 23 Sep '19

23 Sep '19

sujatha <sujatha.sivakumar(a)mariadb.com> writes: > revision-id: e07caf401c26cf8144899336d103e4c7aafd3d7a (mariadb-10.1.41-45-ge07caf401c2) > MDEV-20645: Replication consistency is broken as workers miss the error notification from an earlier failed group. Great that you could come up with a testcase like this to trigger the error! Also, thanks for the detailed description of the issue in the commit mail, that made it much easier to understand and comment on. - Kristian.

1 0

[Maria-developers] Missing memory barrier in parallel replication error handler in wait_for_prior_commit()?
by Kristian Nielsen 22 Sep '19

22 Sep '19

Hi Andrei (Cc: Sujatha), I noticed another thing with the wait_for_commit error handling while looking at the MDEV-18648 patch. We have code like this: // Wakeup code in wait_for_commit::wakeup(): mysql_mutex_lock(&LOCK_wait_commit); waitee= NULL; this->wakeup_error= wakeup_error; // Wait code in wait_for_prior_commit(): if (waitee) return wait_for_prior_commit2(thd); else { if (wakeup_error) my_error(ER_PRIOR_COMMIT_FAILED, MYF(0)); So the waiter code runs a "fast path" without locks. It is ok if we race on the assignment of NULL to wait_for_commit::waitee variable, because then the waiter will take the slow path and do proper locking. But it looks like there is a race as follows: 1. wakeup() sets waitee= NULL 2. wait_for_prior_commit() sees waitee==NULL _and_ wakeup_error==0, and incorrectly returns without error. 3. wakeup() too late sets wait_for_commit::wakeup_error. It is not enough of course to swap the assignments in wakeup(). A write-write memory barrier is needed between them in wakeup(), and a corresponding read-read barrier is needed in wait_for_prior_commit(). With proper barriers, the waiter cannot see the write of waitee=NULL without also seeing the write to wakeup_error. So it will either return with non-zero wakeup_error or take the slow path with proper locking. Both of which are fine. Andrei, what do you think? Can you see something in the above analysis that I missed? It seems like such an obvious miss in the code that I wonder how it remained undetected for so long (or how I could have written that in the first place...). But I suppose the race is very unlikely to hit in practice, especially in a code path that only triggers in the error case... - Kristian.

1 0

Re: [Maria-developers] [Commits] cde9170709c: MDEV-18648: slave_parallel_mode= optimistic default in 10.5
by Kristian Nielsen 22 Sep '19

22 Sep '19

sujatha <sujatha.sivakumar(a)mariadb.com> writes: > Thank you for the review comments. You are right. Setting > rgi->worker_error=1 > for the 2 is the right way to handle. With this, upon reaching > 'finish_event_group' > 2nd will notify 3rd transaction that something went wrong during prior > commit > execution. Ok, great if this fix works. >> Also, this fix doesn't seem to belong with the other MDEV-18648 changes, it >> is unrelated to what the default parallel replication mode is. So please do >> it in a separate commit. >The missing transaction issue is possible, only in the case of >slave_parallel_mode='optimistic'. I *think* it is also possible in conservative mode. Not between different batches of transactions that group-committed together on the master, as you described. But between transactions in one gco. Though I did not check the code deeply for this. But my point was actually that this problem must exist also in earlier versions of MariaDB, where optimistic is not the default, but the user manually enables it. So it should be considered for fixing at least in 10.4 I would assume? - Kristian.

1 0

Re: [Maria-developers] c3d2998a038: MDEV-16130 wrong error message adding AS ROW START to versioned table
by Sergei Golubchik 21 Sep '19

21 Sep '19

Hi, Aleksey! On Aug 30, Aleksey Midenkov wrote: > revision-id: c3d2998a038 (versioning-1.0.3-71-gc3d2998a038) > parent(s): 611488e3d90 > author: Aleksey Midenkov <midenok(a)gmail.com> > committer: Aleksey Midenkov <midenok(a)gmail.com> > timestamp: 2018-05-23 22:45:08 +0300 > message: > > MDEV-16130 wrong error message adding AS ROW START to versioned table > > Closes tempesta-tech/mariadb#494 > > --- > mysql-test/suite/versioning/r/alter.result | 12 ++++++++---- > mysql-test/suite/versioning/t/alter.test | 7 ++++++- > sql/handler.cc | 3 ++- > sql/share/errmsg-utf8.txt | 2 +- > sql/sql_table.cc | 6 ------ > 5 files changed, 17 insertions(+), 13 deletions(-) > > diff --git a/mysql-test/suite/versioning/r/alter.result b/mysql-test/suite/versioning/r/alter.result > index fafcf3c30b0..666420dc2e5 100644 > --- a/mysql-test/suite/versioning/r/alter.result > +++ b/mysql-test/suite/versioning/r/alter.result > @@ -76,13 +76,17 @@ t CREATE TABLE `t` ( > `a` int(11) DEFAULT NULL > ) ENGINE=MyISAM DEFAULT CHARSET=latin1 > alter table t add column trx_start timestamp(6) as row start; > -ERROR HY000: Table `t` is not system-versioned > +ERROR HY000: Can not add system property AS ROW START/END for field `trx_start` This is very strange wording. What is a "system property"? The standard has no such concept. Neither does MariaDB, it's not used anywhere in the manual, as far as I know. Here you can use, for example, ER_VERS_DUPLICATE_ROW_START_END: Duplicate ROW START column `trx_start` > alter table t add system versioning; > show create table t; > Table Create Table > t CREATE TABLE `t` ( > `a` int(11) DEFAULT NULL > ) ENGINE=MyISAM DEFAULT CHARSET=latin1 WITH SYSTEM VERSIONING > +alter table t add column trx_start timestamp(6) as row start; > +ERROR HY000: Can not add system property AS ROW START/END for field `trx_start` > +alter table t modify a int as row start; > +ERROR HY000: Can not add system property AS ROW START/END for field `a` > alter table t add column b int; > show create table t; > Table Create Table > @@ -527,6 +531,6 @@ use test; > # MDEV-15956 Strange ER_UNSUPPORTED_ACTION_ON_GENERATED_COLUMN upon ALTER on versioning column > create or replace table t1 (i int, j int as (i), s timestamp(6) as row start, e timestamp(6) as row end, period for system_time(s,e)) with system versioning; > alter table t1 modify s timestamp(6) as row start; > -ERROR HY000: Can not change system versioning field `s` > +ERROR HY000: Can not add system property AS ROW START/END for field `s` This doesn't look right either. The statement does not add anything, the field `s` is already AS ROW START. Why is it an error at all? > drop database test; > create database test; Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 1

Re: [Maria-developers] [Commits] cde9170709c: MDEV-18648: slave_parallel_mode= optimistic default in 10.5
by Kristian Nielsen 20 Sep '19

20 Sep '19

sujatha <sujatha.sivakumar(a)mariadb.com> writes: Hi Sujutha, > @sql/sql_class.h > Moved 'wait_for_prior_commit(THD *thd)' method inside sql_class.cc > > @sql/sql_class.cc > Added code to check for 'stop_on_error_sub_id' for event groups which get skipped > and don't have any preceding group to wait for. This looks like the wrong fix. The wait_for_commit mechanism is a low-level API, it should not deal with things like stop_on_error_sub_id. > 1,2,3 are scheduled in parallel. > Since the above is true 'skip_event_group=true' is set. Simply call > 'wait_for_prior_commit' to wakeup all waiters. Group '2' didn't had any > waitee and its execution is skipped. Hence its wakeup_error=0.It sends a > positive wakeup signal to '3'. Which commits. This results in a missed > transaction. i.e 33 is missed. I think this the the real problem. 2 is stopping because of error, it must not call wakeup_subsequent_commits() without propagating the error, that breaks the whole semantics of wait_for_commit. Maybe it's enough just to also set rgi->worker_error here, when skip_event_group is set due to error in an earlier group? if (unlikely(entry->stop_on_error_sub_id <= rgi->wait_commit_sub_id)) skip_event_group= true; Then wakeup_subsequent_commits() will correctly inform the following transaction about the error so it doesn't try to commit on its own. There is already similar code in the retry_event_group() function: if (entry->stop_on_error_sub_id == (uint64) ULONGLONG_MAX || rgi->gtid_sub_id < entry->stop_on_error_sub_id) ... else { err= rgi->worker_error= 1; Then all of the changes to sql_class.h and sql_class.cc can be omittet. Also, this fix doesn't seem to belong with the other MDEV-18648 changes, it is unrelated to what the default parallel replication mode is. So please do it in a separate commit. Hope this helps, - Kristian. > diff --git a/sql/sql_class.cc b/sql/sql_class.cc > index 4eab241232b..5ba9c5fe456 100644 > --- a/sql/sql_class.cc > +++ b/sql/sql_class.cc > @@ -7365,6 +7365,33 @@ wait_for_commit::register_wait_for_prior_commit(wait_for_commit *waitee) > } > > > +int wait_for_commit::wait_for_prior_commit(THD *thd) > +{ > + /* > + Quick inline check, to avoid function call and locking in the common case > + where no wakeup is registered, or a registered wait was already signalled. > + */ > + if (waitee) > + return wait_for_prior_commit2(thd); > + else > + { > + if (wakeup_error) > + my_error(ER_PRIOR_COMMIT_FAILED, MYF(0)); > + else > + { > + rpl_group_info* rgi= thd->rgi_slave; > + if (rgi && rgi->is_parallel_exec && > + rgi->parallel_entry->stop_on_error_sub_id < (uint64)ULONGLONG_MAX && > + rgi->gtid_sub_id >= rgi->parallel_entry->stop_on_error_sub_id) > + { > + my_error(ER_PRIOR_COMMIT_FAILED, MYF(0)); > + wakeup_error= ER_PRIOR_COMMIT_FAILED; > + } > + } > + return wakeup_error; > + } > +} > + > /* > Wait for commit of another transaction to complete, as already registered > with register_wait_for_prior_commit(). If the commit already completed, > @@ -7387,6 +7414,17 @@ wait_for_commit::wait_for_prior_commit2(THD *thd) > { > if (wakeup_error) > my_error(ER_PRIOR_COMMIT_FAILED, MYF(0)); > + else > + { > + rpl_group_info* rgi= thd->rgi_slave; > + if (rgi && rgi->is_parallel_exec && > + rgi->parallel_entry->stop_on_error_sub_id < (uint64)ULONGLONG_MAX && > + rgi->gtid_sub_id >= rgi->parallel_entry->stop_on_error_sub_id) > + { > + my_error(ER_PRIOR_COMMIT_FAILED, MYF(0)); > + wakeup_error= ER_PRIOR_COMMIT_FAILED; > + } > + } > goto end; > } > /* > diff --git a/sql/sql_class.h b/sql/sql_class.h > index 72cb8148895..0a0a1aa9fa1 100644 > --- a/sql/sql_class.h > +++ b/sql/sql_class.h > @@ -2062,21 +2062,6 @@ struct wait_for_commit > bool commit_started; > > void register_wait_for_prior_commit(wait_for_commit *waitee); > - int wait_for_prior_commit(THD *thd) > - { > - /* > - Quick inline check, to avoid function call and locking in the common case > - where no wakeup is registered, or a registered wait was already signalled. > - */ > - if (waitee) > - return wait_for_prior_commit2(thd); > - else > - { > - if (wakeup_error) > - my_error(ER_PRIOR_COMMIT_FAILED, MYF(0)); > - return wakeup_error; > - } > - } > void wakeup_subsequent_commits(int wakeup_error_arg) > { > /* > @@ -2123,6 +2108,7 @@ struct wait_for_commit > > void wakeup(int wakeup_error); > > + int wait_for_prior_commit(THD *thd); > int wait_for_prior_commit2(THD *thd); > void wakeup_subsequent_commits2(int wakeup_error); > void unregister_wait_for_prior_commit2();

1 0

[Maria-developers] mdev-19956 - error messages
by Alexander Barkov 19 Sep '19

19 Sep '19

Hi Igor, We discussed the problem with error messages with Sergei and both agreed that a new error message like: Subquery is not allowed in '%s' will be more informative, and it's easy to do. Why not replace: bool expr_allows_subselect; to const char *clause_that_disallows_subselect; ? So we can change the grammar to something like this: | opt_generated_always AS { Lex->clause_that_disallows_subselect= "GENERATED ALWAYS AS"; } virtual_column_func { Lex->clause_that_disallows_subselect= NULL; } The same for other places where we disallow subselects. This can be done in 10.4 before your patch, as a separate commit.

1 0

Re: [Maria-developers] Please review mdev-19956
by Alexander Barkov 19 Sep '19

19 Sep '19

Hi Igor, I have the following suggestions so far. Can you please fix them and replace the commit in bb-10.4-igor? 1. In statements where subquery is not allowed, the error message become confusing: EXECUTE stmt USING (SELECT 1); < ERROR 1064 (42000): ... syntax to use near 'SELECT 1)' at line 1 > ERROR 1064 (42000): ... syntax to use near ')' at line 1 Please change SELECT and WITH to have the '<kwd>' type, like this: %token <kwd> SELECT_SYM /* SQL-2003-R */ %token <kwd> WITH /* SQL-2003-R */ Then in the grammar like this: query_specification_start: SELECT_SYM { SELECT_LEX *sel; LEX *lex= Lex; if (!(sel= lex->alloc_select(TRUE)) || lex->push_select(sel)) MYSQL_YYABORT; sel->init_select(); sel->braces= FALSE; you will be able to use $1.pos() to get the position of the keyword, and later use this position to generate the error properly. 2. The change in sql_cte.cc is not relevant to this task. Please make a separate MDEV for this and push it to 10.4, then rebase MDEV-19956. 3. There are a few mistakes in comments: + <query expression body> :: + <query term> + | <query expression body> UNION [ ALL | DISTINCT ] <query term> + | <query expression body> EXCEPT [ DISTINCT ] + Consider the production rule of the SQL Standard + subquery: + '(' query_expression_no_with_clause ')' + The latter can be re-written into + subquery: + '(' query_expression_body_ext_parens ')' + | '(' with_clause query_expression_no_with_clause ')' Please fix them as discussed on slack. Thanks.

1 0

[Maria-developers] Implementation of Threadpool
by GUESNET, ETIENNE (ext) 18 Sep '19

18 Sep '19

Hi, I am implementing a threadpool system to AIX. The AIX equivalent of epoll / kqueue on AIX is pollset (and IOCP, but partial implementation only). However, pollset has only a level-trigger mode and MariaDB needs edge-trigger (see comments of sql/threadpoll_generic.h file). Adding a pollset support in MariaDB would be difficult, and probably not so efficient, as we need to simulate the edge-trigger behavior. Obviously, AIX has poll and select support. MariaDB has not. Is there a reason to don't implement threadpoll through poll or select? No interest? Performance issues? MariaDB currently works on AIX without threadpool; in term of efficiency, do you know what can be obtained using threadpool with poll/select or a more modern solution? As far I know, SunOS/Solaris/Illumos threadpoll system (called "port") is also level-trigger only, but I do not find specific functions to manage this. Thanks! Etienne Guesnet.

4 6

[Maria-developers] Congratulations on a successful GSoC!
by Vicențiu Ciorbaru 05 Sep '19

05 Sep '19

Hi! By now, all of you have probably received the results of this year's Google Summer of Code. I want to congratulate you all for the work that you have done this year, students and mentors alike. We have had 4 projects this year: 1. EXCEPT ALL and INTERSECT ALL operations - Ruihang Xia https://github.com/MariaDB/server/pull/1378 2. INSERT ... RETURNING - Rucha Deodar https://medium.com/@ruchad1998/google-summer-of-code-2019-add-returning-to-… 3. UPDATE ... RETURNING - Miroslav Koberskii https://gist.github.com/Mup0c/43c781e2135e55bf5126e0f8c60e2a24 4. Support for indexes on expressions - Alexey Mogilyovkin https://gist.github.com/ZeroICQ/f0c35a20cae7065368dae31d7ad9b001 Implementing these projects certainly took a lot of effort. The first one is already merged within MariaDB and the rest will probably follow, after a bit more polishing. The MariaDB community would be grateful if you stuck around to watch over your projects, extend them and improve them over time. I am sure there was a lot to learn from this experience. Now that you have gotten a feel to what it means to contribute significant features to MariaDB you are in a great position to help others! In this sense, I encourage you to keep being active in the community and share what you have learned. :) Thank you for a great summer! Vicențiu Ciorbaru GSoC MariaDB Foundation Admin

1 0

[Maria-developers] Review for MDEV-19708 RBR replication loses data silently ignoring important column attributes
by Alexander Barkov 02 Sep '19

02 Sep '19

Hi Sachin, I've reviewed the code in bb-10.5-19708. It looks good. I suggest some cleanups. - Remove Field::binlog_type_info_slave, it's not used. - Remove Field::binlog_type_info_master. Please don't cache Binlog_type_info inside the Field itself. Let the caller code cache it in an array of Binlog_type_info pointers. - Change this method in Field: virtual Binlog_type_info *binlog_type_info(); to virtual Binlog_type_info *binlog_type_info(MEM_ROOT *) const; Let's pass mem_root as a parameter instead of using table->mem_root. Please add the 'const' qualifier! - Restore the 'const' qualifier to Field::enum_conv_type rpl_conv_type_from - Instead of doing this: Binlog_type_info * Field_new_decimal::binlog_type_info() { Field_num::binlog_type_info(); this->binlog_type_info_master->m_metadata= precision + (decimals() << 8); this->binlog_type_info_master->m_metadata_size= 2; return this->binlog_type_info_master; } Please add a number of Binlog_type_info constructor helpers, in addition to the current one. For example, for numeric types this would be good: Binlog_type_info(uchar type_code, uint16 metadata, uint8 metadata_size, binlog_signess_t signess) :m_type_code(type_code), m_metadata(metadata), m_metadata_size(metadata_size), m_signess(signess), m_cs(NULL), m_enum_typelib(NULL), m_set_typelib(NULL), m_geom_type(GEOM_GEOMETRY) { }; so you can do something simple like this: Binlog_type_info Field_new_decimal::binlog_type_info(MEM_ROOT *root) const { return new (root) Binlog_type_info(type(), precision + (decimals() << 8), 2, signess_arg); } - It seems you always set signess to SIGNESS_NOT_RELEVANT. Where is signess set to SIGNED or UNSIGNED? - This is something we've never used before: +#include <vector> +#include <string> +#include <functional> +#include <memory> We need to discuss this with Serg. Is it possible to avoid this? - There is a new test file mysql-test/suite/rpl/t/rpl_mdev_19708.test.diff but I could not find a corresponding .result.diff. Where is it? - Please use error names instead of numbers in tests: Instead of: --error 1231 SET GLOBAL binlog_row_metadata = -1; it should be: --error ER_WRONG_VALUE_FOR_VAR SET GLOBAL binlog_row_metadata = -1; You can find error names in include/mysqld_ername.h Thanks.

2 1

Re: [Maria-developers] 899c0b3ec6e: Part2: MDEV-18156 Assertion `0' failed or `btr_validate_index(index, 0, false)' in row_upd_sec_index_entry or error code 126: Index is corrupted upon DELETE with PAD_CHAR_TO_FULL_LENGTH
by Sergei Golubchik 30 Aug '19

30 Aug '19

Hi, Alexander! On Aug 30, Alexander Barkov wrote: > revision-id: 899c0b3ec6e (mariadb-10.2.26-51-g899c0b3ec6e) > parent(s): e4415549e53 > author: Alexander Barkov <bar(a)mariadb.com> > committer: Alexander Barkov <bar(a)mariadb.com> > timestamp: 2019-08-29 12:35:19 +0400 > message: > > Part2: MDEV-18156 Assertion `0' failed or `btr_validate_index(index, 0, false)' in row_upd_sec_index_entry or error code 126: Index is corrupted upon DELETE with PAD_CHAR_TO_FULL_LENGTH > > This patch allows the server to open old tables that have > "bad" generated columns (i.e. indexed virtual generated columns, > persistent generated columns) that depend on sql_mode, > for general things like SELECT, INSERT, DROP, etc. > Warning are issued in such cases. > > Only these commands are now disallowed and return an error: > - CREATE TABLE introducing a "bad" generated column > - ALTER TABLE introducing a "bad" generated column > - CREATE INDEX introdicing a "bad" generated column > (i.e. adding an index on a virtual generated column > that depends on sql_mode). > > Note, these commands are allowed: > - ALTER TABLE removing a "bad" generate column > - ALTER TABLE removing an index from a "bad" virtual generated column > - DROP INDEX removing an index from a "bad" virtual generated column > but only if the table does not have any "bad" columns as a result. > > diff --git a/sql/field.cc b/sql/field.cc > index e2b745743d2..b720db19ebf 100644 > --- a/sql/field.cc > +++ b/sql/field.cc > @@ -1428,16 +1428,24 @@ void Field::load_data_set_value(const char *pos, uint length, > } > > > -void Field::error_generated_column_function_is_not_allowed(THD *thd) const > +void Field::error_generated_column_function_is_not_allowed(THD *thd, > + bool error) const > { > StringBuffer<64> tmp; > vcol_info->expr->print(&tmp, (enum_query_type) > (QT_TO_SYSTEM_CHARSET | > QT_ITEM_IDENT_SKIP_DB_NAMES | > QT_ITEM_IDENT_SKIP_TABLE_NAMES)); > - my_error(ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED, MYF(0), > - tmp.c_ptr(), vcol_info->get_vcol_type_name(), > - const_cast<const char*>(field_name)); > + if (error) > + my_error(ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED, MYF(0), > + tmp.c_ptr(), vcol_info->get_vcol_type_name(), > + const_cast<const char*>(field_name)); > + else > + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, > + ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED, > + ER_THD(thd, ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED), > + tmp.c_ptr(), vcol_info->get_vcol_type_name(), > + const_cast<const char*>(field_name)); an easier way of doing it would be my_error(ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED, MYF(error ? 0 : ME_WARNING), tmp.c_ptr(), vcol_info->get_vcol_type_name(), const_cast<const char*>(field_name)); > } > > > diff --git a/sql/table.h b/sql/table.h > index 7786679982f..23a61345242 100644 > --- a/sql/table.h > +++ b/sql/table.h > @@ -325,6 +325,20 @@ enum tmp_table_type > }; > enum release_type { RELEASE_NORMAL, RELEASE_WAIT_FOR_DROP }; > > + > +enum vcol_init_mode > +{ > + VCOL_INIT_DEPENDENCY_FAILURE_IS_WARNING= 1, > + VCOL_INIT_DEPENDENCY_FAILURE_IS_ERROR= 2 > + /* > + There will be a new flags soon, better say "There may be new flags here" because may be there won't be any new flags soon after all, if this your fix will be sufficient :) > + e.g. to automatically remove sql_mode dependency: > + GENERATED ALWAYS AS (char_col) -> > + GENERATED ALWAYS AS (RTRIM(char_col)) > + */ > +}; > + > + > enum enum_vcol_update_mode > { > VCOL_UPDATE_FOR_READ= 0, Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 5ba0b11c5d1: MDEV-20257 Fix crash in Grant_table_base::init_read_record
by Sergei Golubchik 29 Aug '19

29 Aug '19

Hi, Robert! On Aug 28, Robert Bindar wrote: > revision-id: 5ba0b11c5d1 (mariadb-10.4.7-35-g5ba0b11c5d1) > parent(s): 4a490d1a993 > author: Robert Bindar <robert(a)mariadb.org> > committer: Robert Bindar <robert(a)mariadb.org> > timestamp: 2019-08-27 11:38:52 +0300 > message: > > MDEV-20257 Fix crash in Grant_table_base::init_read_record > > The bug shows up when a 10.4+ server starts on a <10.4 datadir > with a crashed mysql.user table. > Grant_tables::open_and_lock tries to open the list of requested > tables (host, db, global_priv,..) and because global_priv doesn't > exist (pre-10.4 datadir), it falls back to opening mysql.user. > The first open_tables call for [host, db, global_priv,..] works > just fine. The second open_tables call (trying to open mysql.user > alone) sees the crashed user table and performs 3 steps: > - closes all the open tables > - attempts a table fix for mysql.user (succeeds) > - opens the tables initially passed as arguments > In an ideal world, the first step should only close the tables > passed as arguments (i.e. mysql.user), but for some reasons > close_tables_for_reopen makes a close_thread_tables call which > closes everything in THD::open_tables (nevertheless, this is > probably the intended behavior). > This side effect causes all the tables opened in the first > open_tables call to now be closed and Grant_table_base::init_read_record > tries to read from a released table. > > diff --git a/mysql-test/main/mdev20257.test b/mysql-test/main/mdev20257.test > new file mode 100644 > index 00000000000..8506292d57c > --- /dev/null > +++ b/mysql-test/main/mdev20257.test > @@ -0,0 +1,51 @@ > +--source include/not_embedded.inc > +--echo # > +--echo # MDEV 20257 Server crashes in Grant_table_base::init_read_record > +--echo # upon crash-upgrade > +--echo # > + > +let $MYSQLD_DATADIR= `select @@datadir`; > + > +rename table mysql.global_priv to mysql.global_priv_bak; > +rename table mysql.user to mysql.user_bak; > +rename table mysql.db to mysql.db_bak; > +rename table mysql.proxies_priv to mysql.proxies_priv_bak; > +rename table mysql.roles_mapping to mysql.roles_mapping_bak; > + > +--source include/shutdown_mysqld.inc > + > +# Bring in a crashed user table > +# Ideally we should've copied only the crashed mysql.user, but this > +# would make mysqld crash in some other place before hitting the > +# crash spot described in MDEV-20257 which is what we're trying to > +# test against. I don't understand that. Where does it crash then? What difference does it make that you replace all tables - are they all corrupted? > +--copy_file std_data/mdev20257/user.frm $MYSQLD_DATADIR/mysql/user.frm > +--copy_file std_data/mdev20257/user.MYD $MYSQLD_DATADIR/mysql/user.MYD > +--copy_file std_data/mdev20257/user.MYI $MYSQLD_DATADIR/mysql/user.MYI > + > +--copy_file std_data/mdev20257/host.frm $MYSQLD_DATADIR/mysql/host.frm > +--copy_file std_data/mdev20257/host.MYD $MYSQLD_DATADIR/mysql/host.MYD > +--copy_file std_data/mdev20257/host.MYI $MYSQLD_DATADIR/mysql/host.MYI > + > +--copy_file std_data/mdev20257/db.frm $MYSQLD_DATADIR/mysql/db.frm > +--copy_file std_data/mdev20257/db.MYD $MYSQLD_DATADIR/mysql/db.MYD > +--copy_file std_data/mdev20257/db.MYI $MYSQLD_DATADIR/mysql/db.MYI > +--copy_file std_data/mdev20257/db.opt $MYSQLD_DATADIR/mysql/db.opt > + > +--source include/start_mysqld.inc > + > +call mtr.add_suppression("Table \'\..mysql\.user\' is marked as crashed and should be repaired"); > +call mtr.add_suppression("Checking table: \'\..mysql\.user\'"); > +call mtr.add_suppression("mysql.user: 1 client is using or hasn't closed the table properly"); > +call mtr.add_suppression("Missing system table mysql..*; please run mysql_upgrade to create it"); > + > +# Cleanup > +drop table mysql.user; > +drop table mysql.db; > +drop table mysql.host; > +rename table mysql.global_priv_bak to mysql.global_priv; > +rename table mysql.user_bak to mysql.user; > +rename table mysql.db_bak to mysql.db; > +rename table mysql.proxies_priv_bak to mysql.proxies_priv; > +rename table mysql.roles_mapping_bak to mysql.roles_mapping; > + > diff --git a/mysql-test/std_data/mdev20257/db.opt b/mysql-test/std_data/mdev20257/db.opt > --- /dev/null > +++ b/mysql-test/std_data/mdev20257/db.opt > @@ -0,0 +1,2 @@ > +default-character-set=latin1 > +default-collation=latin1_swedish_ci why is that? > diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc > index 847d2bd777b..e028d084703 100644 > --- a/sql/sql_acl.cc > +++ b/sql/sql_acl.cc > - DBUG_RETURN(res); > + first = build_open_list(tables, which_tables, lock_type, true); > + > + uint counter; > + if (int rv= really_open(thd, first, &counter)) > + DBUG_RETURN(rv); > + > + TABLE *user_table= tables[USER_TABLE].table; > + if ((which_tables & Table_user) && !user_table) > + { > + close_thread_tables(thd); > + first = build_open_list(tables, which_tables, lock_type, false); Hmm... Here you always close/reopen all tables, just to account for the case when the user table is found corrupted. I'd argue that a corrupted user table happens rarely (compared to the non-corrupted), so it makes sense to optimize for the normal use case. That is, only open the user table as before. After it's opened you check if other tables are still opened - if they aren't you can close and reopen everything. This way you only close/reopen all tables if the user table was corrupted. Makes sense? > + if (int rv= really_open(thd, first, &counter)) > + DBUG_RETURN(rv); > + > + p_user_table= &m_user_table_tabular; > + user_table= tables[USER_TABLE + 1].table; > + } Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 1

Re: [Maria-developers] 5f1cd8fe7a5: MDEV-18156 Assertion `0' failed or `btr_validate_index(index, 0, false)' in row_upd_sec_index_entry or error code 126: Index is corrupted upon DELETE with PAD_CHAR_TO_FULL_LENGTH
by Sergei Golubchik 28 Aug '19

28 Aug '19

Hi, Alexander! On Aug 28, Alexander Barkov wrote: > revision-id: 5f1cd8fe7a5 (mariadb-10.2.26-50-g5f1cd8fe7a5) > parent(s): 9de2e60d749 > author: Alexander Barkov <bar(a)mariadb.com> > committer: Alexander Barkov <bar(a)mariadb.com> > timestamp: 2019-08-26 15:28:32 +0400 > message: > > MDEV-18156 Assertion `0' failed or `btr_validate_index(index, 0, false)' in row_upd_sec_index_entry or error code 126: Index is corrupted upon DELETE with PAD_CHAR_TO_FULL_LENGTH > This is not a trivial one-liner fix. Before pushing, please add a comment explaning the problem and your fix. > > diff --git a/mysql-test/suite/vcol/r/vcol_sql_mode.result b/mysql-test/suite/vcol/r/vcol_sql_mode.result > new file mode 100644 > index 00000000000..6af46dffb80 > --- /dev/null > +++ b/mysql-test/suite/vcol/r/vcol_sql_mode.result > @@ -0,0 +1,234 @@ > +# > +# Start of 10.2 tests > +# > +# > +# MDEV-18156 Assertion `0' failed or `btr_validate_index(index, 0, false)' in row_upd_sec_index_entry or error code 126: Index is corrupted upon DELETE with PAD_CHAR_TO_FULL_LENGTH > +# > +# > +# PAD_CHAR_TO_FULL_LENGTH + various virtual column data types > +# > +CREATE TABLE t1 (a CHAR(5), v CHAR(5) AS (a) VIRTUAL, KEY(v)); > +DROP TABLE t1; please add SHOW CREATE after every CREATE. To show what automatic expression rewriting does (or does not) happen here. > +CREATE TABLE t1 (a CHAR(5), v INT AS (a) VIRTUAL, KEY(v)); > +DROP TABLE t1; > +CREATE TABLE t1 (a CHAR(5), v TIME AS (a) VIRTUAL, KEY(v)); > +DROP TABLE t1; > +CREATE TABLE t1 (c CHAR(8), v BINARY(8) AS (c), KEY(v)); > +ERROR HY000: Function or expression '`c`' cannot be used in the GENERATED ALWAYS AS clause of `v` > +SHOW WARNINGS; Better do --enable_warnings at the beginning of the test. It'll serve two purposes - you won't need to do SHOW WARNINGS manually and it'll make clear when there are no warnings after a CREATE. > +Level Code Message > +Error 1901 Function or expression '`c`' cannot be used in the GENERATED ALWAYS AS clause of `v` > +Warning 1105 depends on system variable @@sql_mode value PAD_CHAR_TO_FULL_LENGTH Let's make it a proper sentence: "Expression `c` implicitly depends on the @@sql_mode value PAD_CHAR_TO_FULL_LENGTH" > +CREATE TABLE t1 (a CHAR(5), v BIT(64) AS (a) VIRTUAL, KEY(v)); > +ERROR HY000: Function or expression '`a`' cannot be used in the GENERATED ALWAYS AS clause of `v` > +SHOW WARNINGS; > +Level Code Message > +Error 1901 Function or expression '`a`' cannot be used in the GENERATED ALWAYS AS clause of `v` > +Warning 1105 depends on system variable @@sql_mode value PAD_CHAR_TO_FULL_LENGTH > +CREATE TABLE t1 (a CHAR(5), v VARCHAR(5) AS (a) VIRTUAL, KEY(v)); > +ERROR HY000: Function or expression '`a`' cannot be used in the GENERATED ALWAYS AS clause of `v` > +SHOW WARNINGS; > +Level Code Message > +Error 1901 Function or expression '`a`' cannot be used in the GENERATED ALWAYS AS clause of `v` > +Warning 1105 depends on system variable @@sql_mode value PAD_CHAR_TO_FULL_LENGTH > +CREATE TABLE t1 (a CHAR(5), v TEXT AS (a) VIRTUAL, KEY(v(100))); > +ERROR HY000: Function or expression '`a`' cannot be used in the GENERATED ALWAYS AS clause of `v` > +SHOW WARNINGS; > +Level Code Message > +Error 1901 Function or expression '`a`' cannot be used in the GENERATED ALWAYS AS clause of `v` > +Warning 1105 depends on system variable @@sql_mode value PAD_CHAR_TO_FULL_LENGTH > +# PAD_CHAR_TO_FULL_LENGTH + TRIM resolving dependency > +CREATE TABLE t1 (a CHAR(5), v VARCHAR(5) AS (RTRIM(a)) VIRTUAL, KEY(v)); > +DROP TABLE t1; > +CREATE TABLE t1 (a CHAR(5), v TEXT AS (RTRIM(a)) VIRTUAL, KEY(v(100))); > +DROP TABLE t1; > +CREATE TABLE t1 (a CHAR(5), v VARCHAR(5) AS (TRIM(TRAILING ' ' FROM a)) VIRTUAL, KEY(v)); > +DROP TABLE t1; > +CREATE TABLE t1 (a CHAR(5), v TEXT AS (TRIM(TRAILING ' ' FROM a)) VIRTUAL, KEY(v(100))); > +DROP TABLE t1; > +CREATE TABLE t1 (a CHAR(5), v VARCHAR(5) AS (TRIM(BOTH ' ' FROM a)) VIRTUAL, KEY(v)); > +DROP TABLE t1; > +CREATE TABLE t1 (a CHAR(5), v TEXT AS (TRIM(BOTH ' ' FROM a)) VIRTUAL, KEY(v(100))); > +DROP TABLE t1; > +CREATE TABLE t1 (a CHAR(5), v VARCHAR(5) AS (TRIM(TRAILING NULL FROM a)) VIRTUAL, KEY(v)); > +DROP TABLE t1; > +CREATE TABLE t1 (a CHAR(5), v VARCHAR(5) AS (TRIM(BOTH NULL FROM a)) VIRTUAL, KEY(v)); > +DROP TABLE t1; > +# PAD_CHAR_TO_FULL_LENGTH + TRIM not resolving dependency > +CREATE TABLE t1 (a CHAR(5), v VARCHAR(5) AS (TRIM(LEADING ' ' FROM a)) VIRTUAL, KEY(v)); > +ERROR HY000: Function or expression 'trim(leading ' ' from `a`)' cannot be used in the GENERATED ALWAYS AS clause of `v` nice :) > +SHOW WARNINGS; > +Level Code Message > +Error 1901 Function or expression 'trim(leading ' ' from `a`)' cannot be used in the GENERATED ALWAYS AS clause of `v` > +Warning 1105 depends on system variable @@sql_mode value PAD_CHAR_TO_FULL_LENGTH > +CREATE TABLE t1 (a CHAR(5), v TEXT AS (TRIM(LEADING ' ' FROM a)) VIRTUAL, KEY(v(100))); > +ERROR HY000: Function or expression 'trim(leading ' ' from `a`)' cannot be used in the GENERATED ALWAYS AS clause of `v` > +SHOW WARNINGS; > +Level Code Message > +Error 1901 Function or expression 'trim(leading ' ' from `a`)' cannot be used in the GENERATED ALWAYS AS clause of `v` > +Warning 1105 depends on system variable @@sql_mode value PAD_CHAR_TO_FULL_LENGTH > +CREATE TABLE t1 (a CHAR(5), v VARCHAR(5) AS (TRIM(TRAILING '' FROM a)) VIRTUAL, KEY(v)); > +ERROR HY000: Function or expression 'trim(trailing '' from `a`)' cannot be used in the GENERATED ALWAYS AS clause of `v` > +SHOW WARNINGS; > +Level Code Message > +Error 1901 Function or expression 'trim(trailing '' from `a`)' cannot be used in the GENERATED ALWAYS AS clause of `v` > +Warning 1105 depends on system variable @@sql_mode value PAD_CHAR_TO_FULL_LENGTH > +CREATE TABLE t1 (a CHAR(5), v VARCHAR(5) AS (TRIM(BOTH '' FROM a)) VIRTUAL, KEY(v)); > +ERROR HY000: Function or expression 'trim(both '' from `a`)' cannot be used in the GENERATED ALWAYS AS clause of `v` > +SHOW WARNINGS; > +Level Code Message > +Error 1901 Function or expression 'trim(both '' from `a`)' cannot be used in the GENERATED ALWAYS AS clause of `v` > +Warning 1105 depends on system variable @@sql_mode value PAD_CHAR_TO_FULL_LENGTH > +CREATE TABLE t1 (a CHAR(5), v VARCHAR(5) AS (TRIM(TRAILING 'x' FROM a)) VIRTUAL, KEY(v)); > +ERROR HY000: Function or expression 'trim(trailing 'x' from `a`)' cannot be used in the GENERATED ALWAYS AS clause of `v` > +SHOW WARNINGS; > +Level Code Message > +Error 1901 Function or expression 'trim(trailing 'x' from `a`)' cannot be used in the GENERATED ALWAYS AS clause of `v` > +Warning 1105 depends on system variable @@sql_mode value PAD_CHAR_TO_FULL_LENGTH > +CREATE TABLE t1 (a CHAR(5), v VARCHAR(5) AS (TRIM(BOTH 'x' FROM a)) VIRTUAL, KEY(v)); > +ERROR HY000: Function or expression 'trim(both 'x' from `a`)' cannot be used in the GENERATED ALWAYS AS clause of `v` > +SHOW WARNINGS; > +Level Code Message > +Error 1901 Function or expression 'trim(both 'x' from `a`)' cannot be used in the GENERATED ALWAYS AS clause of `v` > +Warning 1105 depends on system variable @@sql_mode value PAD_CHAR_TO_FULL_LENGTH > +# PAD_CHAR_TO_FULL_LENGTH + TRIM(... non_constant FROM a) > +CREATE TABLE t1 ( > +a CHAR(5), > +b CHAR(5), > +v TEXT AS (TRIM(TRAILING b FROM a)) VIRTUAL, KEY(v(100))); > +ERROR HY000: Function or expression 'trim(trailing `b` from `a`)' cannot be used in the GENERATED ALWAYS AS clause of `v` > +SHOW WARNINGS; > +Level Code Message > +Error 1901 Function or expression 'trim(trailing `b` from `a`)' cannot be used in the GENERATED ALWAYS AS clause of `v` > +Warning 1105 depends on system variable @@sql_mode value PAD_CHAR_TO_FULL_LENGTH > +# PAD_CHAR_TO_FULL_LENGTH + RPAD resolving dependency > +CREATE TABLE t1 (a CHAR(5), v VARCHAR(5) AS (RPAD(a,5,' ')) VIRTUAL, KEY(v)); > +DROP TABLE t1; > +CREATE TABLE t1 (a CHAR(5), v VARCHAR(5) AS (RPAD(a,6,' ')) VIRTUAL, KEY(v)); > +DROP TABLE t1; > +CREATE TABLE t1 (a CHAR(5), v VARCHAR(5) AS (RPAD(a,6,NULL)) VIRTUAL, KEY(v)); > +DROP TABLE t1; > +CREATE TABLE t1 (a CHAR(5), v VARCHAR(5) AS (RPAD(a,NULL,' ')) VIRTUAL, KEY(v)); > +DROP TABLE t1; > +# PAD_CHAR_TO_FULL_LENGTH + RPAD not resolving dependency > +CREATE TABLE t1 (a CHAR(5), v VARCHAR(5) AS (RPAD(a,4,' ')) VIRTUAL, KEY(v)); > +ERROR HY000: Function or expression 'rpad(`a`,4,' ')' cannot be used in the GENERATED ALWAYS AS clause of `v` > +SHOW WARNINGS; > +Level Code Message > +Error 1901 Function or expression 'rpad(`a`,4,' ')' cannot be used in the GENERATED ALWAYS AS clause of `v` > +Warning 1105 depends on system variable @@sql_mode value PAD_CHAR_TO_FULL_LENGTH > +CREATE TABLE t1 ( > +a CHAR(5), > +b CHAR(5), > +v VARCHAR(5) AS (RPAD(a,NULL,b)) VIRTUAL, > +KEY(v) > +); > +ERROR HY000: Function or expression 'rpad(`a`,NULL,`b`)' cannot be used in the GENERATED ALWAYS AS clause of `v` > +SHOW WARNINGS; > +Level Code Message > +Error 1901 Function or expression 'rpad(`a`,NULL,`b`)' cannot be used in the GENERATED ALWAYS AS clause of `v` > +Warning 1105 depends on system variable @@sql_mode value PAD_CHAR_TO_FULL_LENGTH > +# PAD_CHAR_TO_FULL_LENGTH + comparison > +CREATE TABLE t1 (a CHAR(5), v INT AS (a='a') VIRTUAL, KEY(v)); > +DROP TABLE t1; > +CREATE TABLE t1 ( > +a CHAR(5) CHARACTER SET latin1 COLLATE latin1_nopad_bin, > +v INT AS (a='a') VIRTUAL, KEY(v) > +); > +ERROR HY000: Function or expression '`a` = 'a'' cannot be used in the GENERATED ALWAYS AS clause of `v` > +SHOW WARNINGS; > +Level Code Message > +Error 1901 Function or expression '`a` = 'a'' cannot be used in the GENERATED ALWAYS AS clause of `v` > +Warning 1105 depends on system variable @@sql_mode value PAD_CHAR_TO_FULL_LENGTH > +# PAD_CHAR_TO_FULL_LENGTH + LIKE > +CREATE TABLE t1 (a CHAR(5), v INT AS (a LIKE 'a%') VIRTUAL, KEY(v)); > +DROP TABLE t1; > +CREATE TABLE t1 (a CHAR(5), v INT AS (a LIKE NULL) VIRTUAL, KEY(v)); > +DROP TABLE t1; > +CREATE TABLE t1 (a CHAR(5), v INT AS (a LIKE 'a') VIRTUAL, KEY(v)); > +ERROR HY000: Function or expression '`a` like 'a'' cannot be used in the GENERATED ALWAYS AS clause of `v` > +SHOW WARNINGS; > +Level Code Message > +Error 1901 Function or expression '`a` like 'a'' cannot be used in the GENERATED ALWAYS AS clause of `v` > +Warning 1105 depends on system variable @@sql_mode value PAD_CHAR_TO_FULL_LENGTH > +# > +# Testing NO_UNSIGNED_SUBTRACTION > +# > +CREATE TABLE t1 ( > +a INT UNSIGNED, > +b INT UNSIGNED, > +c INT GENERATED ALWAYS AS (a-b) VIRTUAL, > +KEY (c) > +); > +ERROR HY000: Function or expression '`a` - `b`' cannot be used in the GENERATED ALWAYS AS clause of `c` > +SHOW WARNINGS; > +Level Code Message > +Error 1901 Function or expression '`a` - `b`' cannot be used in the GENERATED ALWAYS AS clause of `c` > +Warning 1105 depends on system variable @@sql_mode value NO_UNSIGNED_SUBTRACTION > +CREATE TABLE t1 ( > +a INT UNSIGNED, > +b INT UNSIGNED, > +c INT GENERATED ALWAYS AS (CAST(a AS SIGNED)-b) VIRTUAL, > +KEY (c) > +); > +ERROR HY000: Function or expression 'cast(`a` as signed) - `b`' cannot be used in the GENERATED ALWAYS AS clause of `c` > +SHOW WARNINGS; > +Level Code Message > +Error 1901 Function or expression 'cast(`a` as signed) - `b`' cannot be used in the GENERATED ALWAYS AS clause of `c` > +Warning 1105 depends on system variable @@sql_mode value NO_UNSIGNED_SUBTRACTION > +CREATE TABLE t1 ( > +a INT UNSIGNED, > +b INT UNSIGNED, > +c INT GENERATED ALWAYS AS (a-CAST(b AS SIGNED)) VIRTUAL, > +KEY (c) > +); > +ERROR HY000: Function or expression '`a` - cast(`b` as signed)' cannot be used in the GENERATED ALWAYS AS clause of `c` > +SHOW WARNINGS; > +Level Code Message > +Error 1901 Function or expression '`a` - cast(`b` as signed)' cannot be used in the GENERATED ALWAYS AS clause of `c` > +Warning 1105 depends on system variable @@sql_mode value NO_UNSIGNED_SUBTRACTION > +CREATE TABLE t1 ( > +a INT UNSIGNED, > +b INT UNSIGNED, > +c INT GENERATED ALWAYS AS (CAST(a AS SIGNED)-CAST(b AS SIGNED)) VIRTUAL, > +KEY (c) > +); > +DROP TABLE t1; > +CREATE TABLE t1 ( > +a INT UNSIGNED, > +b INT UNSIGNED, > +c INT GENERATED ALWAYS AS (CAST(a AS DECIMAL(20,0))-CAST(b AS DECIMAL(20,0))) VIRTUAL, > +KEY (c) > +); > +DROP TABLE t1; > +# > +# Comnination: PAD_CHAR_TO_FULL_LENGTH + NO_UNSIGNED_SUBTRACTION > +# > +CREATE TABLE t1 ( > +a INT UNSIGNED, > +b INT UNSIGNED, > +c CHAR(5), > +v VARCHAR(5) GENERATED ALWAYS AS (RPAD(c,a-b,' ')) VIRTUAL, > +KEY (v) > +); > +ERROR HY000: Function or expression 'rpad(`c`,`a` - `b`,' ')' cannot be used in the GENERATED ALWAYS AS clause of `v` > +SHOW WARNINGS; > +Level Code Message > +Error 1901 Function or expression 'rpad(`c`,`a` - `b`,' ')' cannot be used in the GENERATED ALWAYS AS clause of `v` > +Warning 1105 depends on system variable @@sql_mode value NO_UNSIGNED_SUBTRACTION > +Warning 1105 depends on system variable @@sql_mode value PAD_CHAR_TO_FULL_LENGTH > +CREATE TABLE t1 ( > +a INT UNSIGNED, > +b INT UNSIGNED, > +c CHAR(5), > +v VARCHAR(5) GENERATED ALWAYS AS (RPAD(c,CAST(a AS DECIMAL(20,1))-b,' ')) VIRTUAL, > +KEY (v) > +); > +ERROR HY000: Function or expression 'rpad(`c`,cast(`a` as decimal(20,1)) - `b`,' ')' cannot be used in the GENERATED ALWAYS AS clause of `v` > +SHOW WARNINGS; > +Level Code Message > +Error 1901 Function or expression 'rpad(`c`,cast(`a` as decimal(20,1)) - `b`,' ')' cannot be used in the GENERATED ALWAYS AS clause of `v` > +Warning 1105 depends on system variable @@sql_mode value PAD_CHAR_TO_FULL_LENGTH > +# > +# End of 10.2 tests > +# > diff --git a/sql/field.cc b/sql/field.cc > index e9eaa440952..28caea78c7d 100644 > --- a/sql/field.cc > +++ b/sql/field.cc > @@ -1428,6 +1428,35 @@ void Field::load_data_set_value(const char *pos, uint length, > } > > > +void Field::error_generated_column_function_is_not_allowed(THD *thd) const > +{ > + StringBuffer<64> tmp; > + vcol_info->expr->print(&tmp, QT_TO_SYSTEM_CHARSET); > + my_error(ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED, MYF(0), > + tmp.c_ptr(), vcol_info->get_vcol_type_name(), > + (const char *) field_name /*add .str on merge to 10.3 */); better const_cast<const char*>(field_name) and it won't need a comment - the compiler will catch it > +} > + > + this needs a comment. May be as simple as "see sql_mode.h" btw, you might need this one-line comment in few other places too > +bool Field::check_vcol_sql_mode_dependency(THD *thd) const > +{ > + DBUG_ASSERT(vcol_info); > + if (!(part_of_key.is_clear_all() && key_start.is_clear_all())) 1. why not for persistent? 2. why both part_of_key and key_start? I though part_of_key means "anywhere in the key", that is it's a superset of key_start. > + { > + Sql_mode_dependency dep= > + vcol_info->expr->value_depends_on_sql_mode() & > + Sql_mode_dependency(~0, ~can_handle_sql_mode_dependency_on_store()); > + if (dep) > + { > + error_generated_column_function_is_not_allowed(thd); > + dep.push_dependency_warnings(thd); > + return true; > + } > + } > + return false; > +} > + > + > /** > Numeric fields base class constructor. > */ > diff --git a/sql/item_func.h b/sql/item_func.h > index 81a7e948085..5be62427852 100644 > --- a/sql/item_func.h > +++ b/sql/item_func.h > @@ -773,11 +773,16 @@ class Item_func_plus :public Item_func_additive_op > > class Item_func_minus :public Item_func_additive_op > { > + Sql_mode_dependency m_sql_mode_dependency; > public: > Item_func_minus(THD *thd, Item *a, Item *b): > Item_func_additive_op(thd, a, b) {} > const char *func_name() const { return "-"; } > enum precedence precedence() const { return ADD_PRECEDENCE; } > + Sql_mode_dependency value_depends_on_sql_mode() const > + { > + return m_sql_mode_dependency; 1. no soft_to_hard here? 2. If you do need soft_to_hard here, then, may be, better to do soft_to_hard not in value_depends_on_sql_mode() at all, but just in one place - in Field::check_vcol_sql_mode_dependency() ? > + } > longlong int_op(); > double real_op(); > my_decimal *decimal_op(my_decimal *); > diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc > index 753b6134419..846920bc8c2 100644 > --- a/sql/item_strfunc.cc > +++ b/sql/item_strfunc.cc > @@ -2103,6 +2103,38 @@ void Item_func_trim::print(String *str, enum_query_type query_type) > } > > > +/* > + RTRIM(expr) > + TRIM(TRAILING ' ' FROM expr) > + remove argument's soft dependency on PAD_CHAR_TO_FULL_LENGTH: > +*/ > +Sql_mode_dependency Item_func_trim::value_depends_on_sql_mode() const > +{ > + DBUG_ASSERT(fixed); > + if (arg_count == 1) // RTRIM(expr) > + return (args[0]->value_depends_on_sql_mode() & > + Sql_mode_dependency(~0, ~MODE_PAD_CHAR_TO_FULL_LENGTH)). > + soft_to_hard(); > + // TRIM(... FROM expr) > + DBUG_ASSERT(arg_count == 2); > + if (!args[1]->value_depends_on_sql_mode_const_item()) > + return Item_func::value_depends_on_sql_mode(); > + StringBuffer<64> trimstrbuf; > + String *trimstr= args[1]->val_str(&trimstrbuf); > + if (!trimstr) > + return Sql_mode_dependency(); // will return NULL > + if (trimstr->length() == 0) > + return Item_func::value_depends_on_sql_mode(); // will trim nothing > + if (trimstr->lengthsp() != 0) > + return Item_func::value_depends_on_sql_mode(); // will trim not only spaces what about TRIM(TRAILING ' ' FROM expr) ? it'll trim an even number of spaces, might leave one space untrimmed. > + // TRIM(TRAILING ' ' FROM expr) > + return ((args[0]->value_depends_on_sql_mode() | > + args[1]->value_depends_on_sql_mode()) & > + Sql_mode_dependency(~0, ~MODE_PAD_CHAR_TO_FULL_LENGTH)). > + soft_to_hard(); > +} > + > + > /* Item_func_password */ > > bool Item_func_password::fix_fields(THD *thd, Item **ref) Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 1

Re: [Maria-developers] 25d6f575a50: MDEV-19705: Assertion `tmp >= 0' failed in best_access_path
by Sergei Golubchik 26 Aug '19

26 Aug '19

Hi, Varun! On Aug 26, Varun Gupta wrote: > revision-id: 25d6f575a50 (mariadb-10.4.7-31-g25d6f575a50) > parent(s): 7b4de10477a > author: Varun Gupta <varun.gupta(a)mariadb.com> > committer: Varun Gupta <varun.gupta(a)mariadb.com> > timestamp: 2019-08-22 02:38:38 +0530 > message: > > MDEV-19705: Assertion `tmp >= 0' failed in best_access_path > > The reason for hitting the assert is that rec_per_key estimates have some garbage value. > So the solution to fix this would be for long unique keys to use use rec_per_key for only 1 keypart > > --- > mysql-test/main/long_unique.result | 15 +++++++++++++++ > mysql-test/main/long_unique.test | 14 ++++++++++++++ > sql/table.cc | 13 ++++++++++++- > 3 files changed, 41 insertions(+), 1 deletion(-) > > diff --git a/sql/table.cc b/sql/table.cc > index 1ab4df0f7cf..fa57693e7eb 100644 > --- a/sql/table.cc > +++ b/sql/table.cc > @@ -828,6 +829,16 @@ static bool create_key_infos(const uchar *strpos, const uchar *frm_image_end, > { > keyinfo->key_length= HA_HASH_KEY_LENGTH_WITHOUT_NULL; > key_part++; // reserved for the hash value > + /* > + but hash keys have a flag "only whole key", so for hash keys one should never > + look at keyinfo->rec_per_key for partial keys, and they cannot possibly be calculated > + so here we make sure that the keyinfo->rec_per_key[keyinfo->user_defined_key_parts-1] > + would point to the slot we reserved for the keyinfo which holds the unique key with > + hash > + */ > + ulong *dest= (keyinfo->rec_per_key + keyinfo->user_defined_key_parts - 1); > + memcpy(&dest, &rec_per_key, sizeof(ulong*)); Sorry, I don't understand what you're doing here. I only meant something like keyinfo->rec_per_key-= keyinfo->user_defined_key_parts - 1; > + *rec_per_key++= 0; > } > > /* > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

[Maria-developers] Weekly Report
by Rucha Deodhar 19 Aug '19

19 Aug '19

Report for week 12: Hi! This week I merged some test cases in respective files into one so that the tests can be under 80 lines and also improved the documentation. Documentation for INSERT...RETURNING: https://docs.google.com/document/d/1EknNlh-J72cUlCg9rvcujUeZdzu5uzm0hmHG3qg… REPLACE...RETURNING: https://docs.google.com/document/d/1UMy5fi-j6yIObYIRwTteaJKHU5F-wPoEAVgVbeu… I also fixed feature_insert_returning system variable by adding some code and improving the test file. The test is passing. insert_returning_datatypes.result and replace_returning_datatypes.result files were showing binary data. To fix it, it was suggested to create a .reject and .result files, hexdump of the files, diff and giving a printable ASCII character as the input in BIT field because BIT is printed as VARCHAR. One thing I noticed was, after fixing the files they were still showing as binary files, which was probably because of the diffs. So I removed the existing files from my repo and pushed new files then cleaned up the commits again and fixed the 80 character per line rule for new tests. My github repo is up to date with latest changes. Regards, Rucha Deodhar

1 0

Re: [Maria-developers] e2a82094332: MENT-253 Server Audit v2 does not work with PS protocol: server crashes in filter_query_type.
by Sergei Golubchik 14 Aug '19

14 Aug '19

Hi, Alexey! On Aug 14, Alexey Botchkov wrote: > revision-id: e2a82094332 (mariadb-10.4.7-123-ge2a82094332) > parent(s): 691654721b3 > author: Alexey Botchkov <holyfoot(a)mariadb.com> > committer: Alexey Botchkov <holyfoot(a)mariadb.com> > timestamp: 2019-08-07 11:49:54 +0400 > message: > > MENT-253 Server Audit v2 does not work with PS protocol: server crashes in filter_query_type. > > Set the thd->query_string where possible. > server_audit2 should not crash when gets empty event->query. > > --- > plugin/server_audit2/server_audit.c | 15 ++++++++++++--- > sql/sql_prepare.cc | 22 ++++++++++++++++++++++ > 2 files changed, 34 insertions(+), 3 deletions(-) Test case? With normal and error code paths? > diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc > index 74723d5bd91..d2cb4e9b372 100644 > --- a/sql/sql_prepare.cc > +++ b/sql/sql_prepare.cc > @@ -2600,6 +2600,14 @@ void mysqld_stmt_prepare(THD *thd, const char *packet, uint packet_length) > > if (stmt->prepare(packet, packet_length)) > { > + /* > + Set the thd->query_string with the current query so the > + audit plugin gets the meaningful notification. say that it's an error case. Like Prepare failed. Set the thd->query_string with the current query so the audit plugin gets the meaningful notification when it gets the error notification. > + */ > + if (alloc_query(thd, stmt->query_string.str(), stmt->query_string.length())) > + { > + thd->set_query(0, 0); > + } > /* Statement map deletes statement on erase */ > thd->stmt_map.erase(stmt); > thd->clear_last_stmt(); > @@ -3184,6 +3192,13 @@ static void mysql_stmt_execute_common(THD *thd, > char llbuf[22]; > my_error(ER_UNKNOWN_STMT_HANDLER, MYF(0), static_cast<int>(sizeof(llbuf)), > llstr(stmt_id, llbuf), "mysqld_stmt_execute"); > + /* > + Set thd->query_string with the stmt_id so the > + audit plugin gets the meaningful notification. Same here. > + */ > + if (alloc_query(thd, llbuf, strlen(llbuf))) > + thd->set_query(0, 0); > + > DBUG_VOID_RETURN; > } > stmt->read_types= read_types; > @@ -3939,6 +3954,13 @@ bool Prepared_statement::prepare(const char *packet, uint packet_len) > DBUG_RETURN(TRUE); > } > > + /* > + We'd like to have thd->query to be set to the actual query > + after the function ends. > + This value will be sent to audit pulgins later. 1. typo: "pulgins" 2. Mention explicitly that "query string is allocated in the stmt arena, not in the thd arena. But it's safe here, because stmt always has a longer lifetime than thd arena." - or something along these lines. > + */ > + stmt_backup.query_string= thd->query_string; > + > old_stmt_arena= thd->stmt_arena; > thd->stmt_arena= this; > > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

[Maria-developers] Weekly Report
by Rucha Deodhar 12 Aug '19

12 Aug '19

Report for week 11: Hello! This week I worked on cleaning up the code, commits and test cases. I was a little less familiar with git rebase -i so tried a couple of things on a dummy repo first and then cleaned up the commits on my repo. I also worked on the coding style review and removed all the trailing white spaces and tabs mentioned in the review. To remove the last argument of mysql_prepare_insert() I tried a couple of things like not calling setup_fields() and setup_wild() separately for INSERT...SELECT...RETURNING and calling std::swap if select_insert is true. Like so if(!lex->returning_list.is_empty()) { if (select_insert) //std::swap() setup_fields() and setup_wild() if (selec_insert) //std::swap() } and also changing where the above functions are called. But that didn't work. So now calling it separately for INSERT..SELECT..RETURNING and other variants. I added separate function to sql_base.cc and sql_base.h which calls setup_fields() and setup_wild() so that not only the last argument can be removed but also lines of code can be reduced. All the occurrences of (wild_num && setup_wild(thd, table_list, field_list, NULL, wild_num, &select_lex->hidden_bit_fields)) || setup_fields(thd, Ref_ptr_array(), field_list, MARK_COLUMNS_READ, NULL, NULL, 0) can be replaced with setup_returning_fields(). (Since this is used in insert..returning, insert..select..returning, delete...returning and update...returning too). On some lines of the test cases, the characters were coming to more than 80 characters. So fixed that and added some more test cases (INSERT...IGNORE..RETURNING and fields with auto_increment). Finally I merged all the commits made this week into one commit. My github repo is up to date with the latest changes. Here is the link for the same: https://github.com/rucha174/server/commit/0d7eb4198cc0aadba42acb4960c175702… Regards, Rucha Deodhar

1 0

[Maria-developers] Coding Style Review INSERT RETURNING
by Vicențiu Ciorbaru 07 Aug '19

07 Aug '19

Hi Rucha! This is going to be a very thorough review looking at coding style. I'll nitpick on every line of code where I find something, just be patient and let's clean up everything. You might consider me picking on the coding style a bit pedantic, but it really is important. It is important because it helps in searching, it helps in understanding the code faster, it helps to look-up history faster. The good part is that once you get used to writing things properly, you won't even have to think about it actively. It will just come naturally. I suspect you do not have your editor configured to show whitespaces and show you tabs versus spaces. Please look into your editor's settings to highlight both tabs and spaces. There are scripts that remove trailing whitespaces for you, across whole files. I don't recommend them for MariaDB code as we have remnants from MySQL times of these and because we care more about preserving the immediate history for running `git blame`, we did not fully fix all files. For any other project, I highly recommend you use them. Some of these comments might slightly contradict what I said before. That is because I wanted us to get things working first and I didn't want to burden you with too many details initially. I'll look to explain myself now clearly so you understand why some things are bad ideas. I want you to take this review also as a general guideline for any other code you write, anywhere. With these hints, tricks, ideas, you'll produce much higher quality code in the long run. It will take you longer to write sometimes, but it really makes a difference. :) Also, you may have noticed some parts of our code do not agree with my general guidelines. You have to take into account that this code was written over the course of many years, by different people, with different levels of experience or available time to implement something. Try not to make use of it as an excuse for new code that you write. It's up to you to always write the best possible version *with the time allowed*. We have adequate time now so we can afford to look for better solutions. With all that said, this took me a while to write and I may still have missed a few things. Feel free to point out anything that looks strange. Now, let's get to the review. > > diff --git a/sql/sql_class.h b/sql/sql_class.h > index 18ccf992d1e..9e4bb1d4579 100644 > --- a/sql/sql_class.h > +++ b/sql/sql_class.h > @@ -5475,7 +5476,13 @@ class select_dump :public select_to_file { > > class select_insert :public select_result_interceptor { > public: > + select_result* sel_result; The * should be next to the variable name. Not everybody does this, but the code around it does, so let's be consistent. > TABLE_LIST *table_list; > + /* > + Points to the table in which we're inserting records. We need to save the > + insert table before it is masked in the parsing phase. > + */ > + TABLE_LIST *insert_table; This sort of code and comment is usually a bad idea: 1. We are adding an extra member, because of a hack in a different part of the code. 2. We are mentioning a specific flow for which this member is good for. This already begins to hint at a bad design decision. Sometimes there's no way around it, but this guarantees lack of re-usability. If, in the future we fix the "masking in the parsing phase" to no longer mask the first table, which we probably will do, because we have to to allow INSERT within other queries, the comment will become outdated. One needs to remember this too. Forgetting to update comments happens easily. Here is one example: One checks the class initially, finds the appropriate member to use, uses it in a different context than the original author thought of. The class then is no longer changed so it does not show up during review time, the reviewer doesn't think to check the class member's definition either. You now have outdated comments. To prevent this from happening, avoid as much as possible to explain where a member is used, rather explain what type of data it is supposed to store. The first sentence accomplishes that goal. > TABLE *table; > List<Item> *fields; > ulonglong autoinc_value_of_last_inserted_row; // autogenerated or not > @@ -5484,7 +5491,7 @@ class select_insert :public select_result_interceptor { > select_insert(THD *thd_arg, TABLE_LIST *table_list_par, > TABLE *table_par, List<Item> *fields_par, > List<Item> *update_fields, List<Item> *update_values, > - enum_duplicates duplic, bool ignore); > + enum_duplicates duplic, bool ignore,select_result* sel_ret_list,TABLE_LIST *save_first); This is an artifact of our long-standing code base. TABS mixed with spaces. :( Let's clean up all of this select_insert's constructor. Do not keep the tabs, we are modifying it anyway. Turn all these tabs to proper spaces. Make sure the function arguments match the beginning parenthesis like so: <function_name>(TYPE1 arg1, TYPE2 arg2, TYPE3 arg3, TYPE4 arg4, TYPE5 arg5) > ~select_insert(); > int prepare(List<Item> &list, SELECT_LEX_UNIT *u); > virtual int prepare2(JOIN *join); > @@ -5520,7 +5527,7 @@ class select_create: public select_insert { > List<Item> &select_fields,enum_duplicates duplic, bool ignore, > TABLE_LIST *select_tables_arg): > select_insert(thd_arg, table_arg, NULL, &select_fields, 0, 0, duplic, > - ignore), > + ignore, NULL, NULL), Sigh... again, we have a historical background of mixing NULLs with 0s. Ok here I guess. > create_table(table_arg), > create_info(create_info_par), > select_tables(select_tables_arg), > diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc > index f3a548d7265..4bb19cef726 100644 > --- a/sql/sql_insert.cc > +++ b/sql/sql_insert.cc > @@ -119,6 +119,18 @@ static bool check_view_insertability(THD *thd, TABLE_LIST *view); > @returns false if success. > */ > > +/* > + Swaps the context before and after calling setup_fields() and setup_wild() in > + INSERT...SELECT when LEX::returning_list is not empty. One more space to indent the comment text, for both lines. > +*/ > +template <typename T> > +void swap_context(T& cxt1, T& cxt2) > +{ > + T temp = cxt1; > + cxt1 = cxt2; > + cxt2 = temp; > + > +} Unneded empty line before the the closing }. Also, you should add 2 empty lines after the function. Delete the tabs in the function, replace with 2 spaces. Also, this function is not even used just to swap_contexts any more. Rename it to swap. Can we not use std::swap instead? > static bool check_view_single_update(List<Item> &fields, List<Item> *values, > TABLE_LIST *view, table_map *map, > bool insert) > @@ -688,6 +700,10 @@ Field **TABLE::field_to_fill() > > /** > INSERT statement implementation > + Trailing whitespace. > + SYNOPSIS > + mysql_insert() > + result NULL if returning_list is empty I'd change this to NULL if the insert is not outputing results via 'RETURNING' clause. > > @note Like implementations of other DDL/DML in MySQL, this function > relies on the caller to close the thread tables. This is done in the > @@ -700,7 +716,7 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, > List<Item> &update_fields, > List<Item> &update_values, > enum_duplicates duplic, > - bool ignore) > + bool ignore, select_result* result) Remove tabs, use spaces, align properly, like explained above. > { > bool retval= true; > int error, res; > @@ -719,6 +735,8 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, > List_item *values; > Name_resolution_context *context; > Name_resolution_context_state ctx_state; > + SELECT_LEX* select_lex = thd->lex->first_select_lex(); > + List<Item>& returning_list = thd->lex->returning_list; Generally this kind of use of references is a bad idea. Highly suggest you only use const references, otherwise things can turn very confusing, very quickly. If you read the Google C++ coding style guide, you'll see they only allow const references in any part of their code. > #ifndef EMBEDDED_LIBRARY > char *query= thd->query(); > /* > @@ -775,9 +793,13 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, > > if (mysql_prepare_insert(thd, table_list, table, fields, values, > update_fields, update_values, duplic, &unused_conds, > - FALSE)) > - goto abort; > - > + FALSE,result?true:false)) Missing a few spaces. One after , One before and one after ? One before and one after :. > + goto abort; Tabs instead of spaces. > + Trailing whitespaces. > + /* Prepares LEX::returing_list if it is not empty */ > + if (result) > + (void)result->prepare(returning_list, NULL); Tabs instead of spaces. > + Tabs, spaces, trailing whitespaces. Make sure each new line that contains no text be just that, a newline. > /* mysql_prepare_insert sets table_list->table if it was not set */ > table= table_list->table; > > @@ -947,6 +969,18 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, > goto values_loop_end; > } > } > + /* > + If statement returns result set, we need to send the result set metadata > + to the client so that it knows that it has to expect an EOF or ERROR. > + At this point we have all the required information to send the result set metadata. > + */ When adding comments like these, you need to indent the text 2 more spaces. You crossed the 80 character limit with the "metadata" word. That needs to be put on a new line. > + if (result) > + { > + if (unlikely(result->send_result_set_metadata(returning_list, Tabs again. > + Protocol::SEND_NUM_ROWS | > + Protocol::SEND_EOF))) > + goto values_loop_end; The goto indentation is ok, but Protocol should be aligned to the last open parenthesis. > + } > > THD_STAGE_INFO(thd, stage_update); > do > @@ -1060,7 +1094,7 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, > error= 1; > break; > } > - > + You added an extra line here, but introduced a tab and trailing whitespaces, not good. > #ifndef EMBEDDED_LIBRARY > if (lock_type == TL_WRITE_DELAYED) > { > @@ -1072,9 +1106,25 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, > } > else > #endif > - error=write_record(thd, table ,&info); > + error=write_record(thd, table ,&info); You didn't change this line logic at all. Just introduced an extra tab plus whitespaces. Please clean it up. > if (unlikely(error)) > break; > + /* > + We send the rows after writing them to table so that the correct information > + is sent to the client. Example INSERT...ON DUPLICAT KEY UPDATE and values > + set to auto_increment. Write record handles auto_increment updating values > + if there is a duplicate key. We want to send the rows to the client only > + after these operations are carried out. Otherwise it shows 0 to the client > + if the fields that are incremented automatically are not given explicitly > + and the irreplaced values in case of ON DUPLICATE KEY UPDATE (even if > + the values are replaced or incremented while writing record. > + Hence it shows different result set to the client) > + */ Good comment! Trailing whitespaces however are still present and you are breaking the 80 character rule. > + if (result && result->send_data(returning_list) < 0) > + { > + error = 1; Remove space before =. > + break; > + } > thd->get_stmt_da()->inc_current_row_for_warning(); > } > its.rewind(); > @@ -1233,19 +1283,30 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, > retval= thd->lex->explain->send_explain(thd); > goto abort; > } > + Trailing whitespaces. Please fix. > if ((iteration * values_list.elements) == 1 && (!(thd->variables.option_bits & OPTION_WARNINGS) || > !thd->cuted_fields)) > - { > - my_ok(thd, info.copied + info.deleted + > + { > + /* > + Client expects an EOF/Ok packet if result set metadata was sent. If > + LEX::returning_list is not empty and the statement returns result set > + we send EOF which is the indicator of the end of the row stream. > + Else we send Ok packet i.e when the statement returns only the status > + information > + */ > + if (result) > + result->send_eof(); > + else > + my_ok(thd, info.copied + info.deleted + > ((thd->client_capabilities & CLIENT_FOUND_ROWS) ? > - info.touched : info.updated), > - id); > + info.touched : info.updated),id); > } > else > - { > + { You didn't change the logic here, just added an extra trailing whitespace. > char buff[160]; > ha_rows updated=((thd->client_capabilities & CLIENT_FOUND_ROWS) ? > info.touched : info.updated); > + I guess I am ok with an extra line here, but it should not have an extra tab. Please, only newlines, no extra characters. > if (ignore) > sprintf(buff, ER_THD(thd, ER_INSERT_INFO), (ulong) info.records, > (lock_type == TL_WRITE_DELAYED) ? (ulong) 0 : > @@ -1255,8 +1316,12 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, > sprintf(buff, ER_THD(thd, ER_INSERT_INFO), (ulong) info.records, > (ulong) (info.deleted + updated), > (long) thd->get_stmt_da()->current_statement_warn_count()); > - ::my_ok(thd, info.copied + info.deleted + updated, id, buff); > + if (result) > + result->send_eof(); > + else Tabs, please use spaces. > + ::my_ok(thd, info.copied + info.deleted + updated, id, buff); > } > + Trailing whitespaces. :( > thd->abort_on_warning= 0; > if (thd->lex->current_select->first_cond_optimization) > { > @@ -1470,6 +1535,7 @@ static void prepare_for_positional_update(TABLE *table, TABLE_LIST *tables) > be taken from table_list->table) > where Where clause (for insert ... select) > select_insert TRUE if INSERT ... SELECT statement > + with_returning_list TRUE if returning_list is not empty Trailing whitespace. (I know the rest of the function comments also have trailing whitespaces. Perfect time to fix all that and remove all their tabs. > > TODO (in far future) > In cases of: > @@ -1490,7 +1556,7 @@ bool mysql_prepare_insert(THD *thd, TABLE_LIST *table_list, > TABLE *table, List<Item> &fields, List_item *values, > List<Item> &update_fields, List<Item> &update_values, > enum_duplicates duplic, COND **where, > - bool select_insert) > + bool select_insert, bool with_returning_list) One too many spaces after the last comma. > { > SELECT_LEX *select_lex= thd->lex->first_select_lex(); > Name_resolution_context *context= &select_lex->context; > @@ -1556,8 +1622,21 @@ bool mysql_prepare_insert(THD *thd, TABLE_LIST *table_list, > */ > table_list->next_local= 0; > context->resolve_in_table_list_only(table_list); > + Trailing whitespaces. > + /* > + Perform checks like all given fields exists, if exists fill struct with Trailing whitespace. > + current data and expand all '*' in given fields for LEX::returning_list. > + */ Trailing whitespaces. > + if(with_returning_list) Trailing whitespaces. Put a space after if. > + { > + res= ((select_lex->with_wild && setup_wild(thd, table_list, Trailing whitespace. > + thd->lex->returning_list,NULL, select_lex->with_wild, Trailing whitespace, tabs instead of spaces. > + &select_lex->hidden_bit_fields)) || > + setup_fields(thd, Ref_ptr_array(), > + thd->lex->returning_list, MARK_COLUMNS_READ, 0, NULL, 0)); Tabs instead of spaces, swap them to spaces. Make sure to follow the 80 character max line length. > + } > > - res= (setup_fields(thd, Ref_ptr_array(), > + res= (res||setup_fields(thd, Ref_ptr_array(), Please Add a space before and after ||. > *values, MARK_COLUMNS_READ, 0, NULL, 0) || > check_insert_fields(thd, context->table_list, fields, *values, > !insert_into_view, 0, &map)); > @@ -1724,7 +1803,7 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info) > table->file->insert_id_for_cur_row= insert_id_for_cur_row; > bool is_duplicate_key_error; > if (table->file->is_fatal_error(error, HA_CHECK_ALL)) > - goto err; > + goto err; A good fix, not strictly necessary, but a good fix. > is_duplicate_key_error= > table->file->is_fatal_error(error, HA_CHECK_ALL & ~HA_CHECK_DUP); > if (!is_duplicate_key_error) > @@ -3557,29 +3636,39 @@ bool Delayed_insert::handle_inserts(void) > TRUE Error > */ > > -bool mysql_insert_select_prepare(THD *thd) > +bool mysql_insert_select_prepare(THD *thd,select_result *sel_res) Put a space after last comma (,). This function could use a bit of tweaking, but let's fix all the code style issues first. > { > LEX *lex= thd->lex; > SELECT_LEX *select_lex= lex->first_select_lex(); > DBUG_ENTER("mysql_insert_select_prepare"); > - > + List<Item>& returning_list = thd->lex->returning_list; For assignment operators no space before it. Like so: List<Item>& returning_list= thd->lex->returning_list; > > > /* > SELECT_LEX do not belong to INSERT statement, so we can't add WHERE > clause if table is VIEW > */ > - > + /* > + Passing with_returning_list (last argument) as false otherwise > + setup_field() and setup_wild() will be called twice. 1) in mysql_prepare_insert() > + and 2) time in select_insert::prepare(). We want to call it only once: > + in select_insert::prepare() > + */ Ok, this is an interesting comment. It's much better if we can write code that prevents this sort of reasoning. Pragmatically, why do we end up calling it twice? Can we somehow refactor it such that this is not a problem in the first place? For example, here it kind of makes sense to just compute with_returning_list within the function, by looking at thd->lex->returning_list. > if (mysql_prepare_insert(thd, lex->query_tables, > lex->query_tables->table, lex->field_list, 0, > lex->update_list, lex->value_list, lex->duplicates, > - &select_lex->where, TRUE)) > + &select_lex->where, TRUE,false)) Put a space after last comma. And let's use FALSE. > DBUG_RETURN(TRUE); > > + /* If LEX::returning_list is not empty, we prepare the list */ Uhm, the comment doesn't seem to match the code. You are checking sel_res, yet the comment says that if LEX:returning_list is not empty, prepare the list. > + if (sel_res) > + (void)sel_res->prepare(returning_list, NULL); Here we have a tab instead of spaces again. > + > DBUG_ASSERT(select_lex->leaf_tables.elements != 0); > List_iterator<TABLE_LIST> ti(select_lex->leaf_tables); > TABLE_LIST *table; > uint insert_tables; > > + This line is not necessary. > if (select_lex->first_cond_optimization) > { > /* Back up leaf_tables list. */ > @@ -3607,6 +3696,7 @@ bool mysql_insert_select_prepare(THD *thd) > while ((table= ti++) && insert_tables--) > ti.remove(); > > + This line is not necessary. > DBUG_RETURN(FALSE); > } > > @@ -3617,7 +3707,7 @@ select_insert::select_insert(THD *thd_arg, TABLE_LIST *table_list_par, > List<Item> *update_fields, > List<Item> *update_values, > enum_duplicates duplic, > - bool ignore_check_option_errors): > + bool ignore_check_option_errors,select_result *result, TABLE_LIST *save_first): This line is still way too long. I remember mentioning this in a previous review, please look more closely and wrap this properly. > select_result_interceptor(thd_arg), > table_list(table_list_par), table(table_par), fields(fields_par), > autoinc_value_of_last_inserted_row(0), > @@ -3630,6 +3720,8 @@ select_insert::select_insert(THD *thd_arg, TABLE_LIST *table_list_par, > info.update_values= update_values; > info.view= (table_list_par->view ? table_list_par : 0); > info.table_list= table_list_par; > + sel_result= result; > + insert_table= save_first; Trailing whitespace, you added a tab after =. Please delete the tab. > } > > > @@ -3637,10 +3729,11 @@ int > select_insert::prepare(List<Item> &values, SELECT_LEX_UNIT *u) > { > LEX *lex= thd->lex; > - int res; > + int res=0; Space after =. > table_map map= 0; > SELECT_LEX *lex_current_select_save= lex->current_select; > DBUG_ENTER("select_insert::prepare"); > + SELECT_LEX* select_lex = thd->lex->first_select_lex(); Delete space before =. > > unit= u; > > @@ -3651,7 +3744,33 @@ select_insert::prepare(List<Item> &values, SELECT_LEX_UNIT *u) > */ > lex->current_select= lex->first_select_lex(); > > - res= (setup_fields(thd, Ref_ptr_array(), > + /* > + We want the returning_list to point to insert table. But the context is masked. > + So we swap it with the context saved during parsing stage. Tab mixed with spaces. Only use spaces. Traling whitespace, here and the line before. > + */ > + if(sel_result) Trailing whitespace. > + { Trailing whitespace. > + swap_context(insert_table,select_lex->table_list.first); > + swap_context(select_lex->context.saved_table_list,select_lex->context.table_list); > + swap_context(select_lex->context.saved_name_resolution_table,select_lex->context.first_name_resolution_table); Line length exceeds 80 characters. > + > + /* > + Perform checks for LEX::returning_list like we do for other variant of INSERT. > + */ > + res=((select_lex->with_wild && setup_wild(thd, table_list, > + thd->lex->returning_list, NULL, select_lex->with_wild, > + &select_lex->hidden_bit_fields)) || > + setup_fields(thd, Ref_ptr_array(), > + thd->lex->returning_list, MARK_COLUMNS_READ, 0, NULL, 0)); Tabs instead of spaces and mixed spaces. Not good. Also trailing whitespaces. Also, the parameters of setup_wild should be aligned to the first parenthesis of setup_wild. setup_fields should be aligned to the first parenthesis here. I also think there's one too many parantheses here, the outer most ones are not necessary.. > + > + /* Swap it back to retore the previous state for the rest of the function */ Tabs instead of spaces. > + > + swap_context(insert_table,select_lex->table_list.first); > + swap_context(select_lex->context.saved_table_list,select_lex->context.table_list); > + swap_context(select_lex->context.saved_name_resolution_table, select_lex->context.first_name_resolution_table); Tabs and spaces mixed here. Watch line length. > + } > + > + res= res || (setup_fields(thd, Ref_ptr_array(), > values, MARK_COLUMNS_READ, 0, NULL, 0) || > check_insert_fields(thd, table_list, *fields, values, > !insert_into_view, 1, &map)); The line that starts with "values" should be aligned to the first open parenthesis. > @@ -3822,13 +3941,26 @@ select_insert::prepare(List<Item> &values, SELECT_LEX_UNIT *u) > > int select_insert::prepare2(JOIN *) > { > + Trailing whitespace. > DBUG_ENTER("select_insert::prepare2"); > + List<Item>& returning_list = thd->lex->returning_list; Delete space before = > + LEX* lex = thd->lex; This function does not use the lex local variable at all, only via thd->lex. This line should be removed. > if (thd->lex->current_select->options & OPTION_BUFFER_RESULT && > thd->locked_tables_mode <= LTM_LOCK_TABLES && > !thd->lex->describe) > table->file->ha_start_bulk_insert((ha_rows) 0); > if (table->validate_default_values_of_unset_fields(thd)) > DBUG_RETURN(1); > + > + /* Same as the other variants of INSERT */ > + if (sel_result) > + { > + if(unlikely(sel_result->send_result_set_metadata(returning_list, > + Protocol::SEND_NUM_ROWS | > + Protocol::SEND_EOF))) The Protocol should be aligned to the last open parenthesis. You are mixing tabs with spaces here. Convert to spaces. > + > + DBUG_RETURN(1); Mixing tabs with spaces again, convert to spaces. > + } > DBUG_RETURN(0); > } > > @@ -3842,6 +3974,8 @@ void select_insert::cleanup() > select_insert::~select_insert() > { > DBUG_ENTER("~select_insert"); > + sel_result=NULL; Add a space after =. > + insert_table=NULL; Add a space after =. > if (table && table->is_created()) > { > table->next_number_field=0; > @@ -3857,6 +3991,8 @@ select_insert::~select_insert() > int select_insert::send_data(List<Item> &values) > { > DBUG_ENTER("select_insert::send_data"); > + LEX* lex = thd->lex; Delete space before =. > > + List<Item>& returning_list = thd->lex->returning_list; Delete space before =. > bool error=0; > > if (unit->offset_limit_cnt) > @@ -3889,8 +4025,18 @@ int select_insert::send_data(List<Item> &values) > DBUG_RETURN(1); > } > } > - > + Trailing whitespaces added for no good reason. > error= write_record(thd, table, &info); > + Trailing whitespaces. > + /* > + Sending the result set to the cliet after writing record. The reason is same > + as other variants of insert. > + */ > + if (sel_result && sel_result->send_data(returning_list) < 0) > + { > + error = 1; Delete space before =. > + DBUG_RETURN(1); > + } > table->vers_write= table->versioned(); > table->auto_increment_field_not_null= FALSE; > > @@ -4024,7 +4170,7 @@ bool select_insert::send_ok_packet() { > char message[160]; /* status message */ > ulonglong row_count; /* rows affected */ > ulonglong id; /* last insert-id */ > - > + LEX *lex=thd->lex; This function does not use this variable at all. Please delete it if it's not used. All it does is produce a compiler warning. > DBUG_ENTER("select_insert::send_ok_packet"); > > if (info.ignore) > @@ -4045,8 +4191,15 @@ bool select_insert::send_ok_packet() { > (thd->arg_of_last_insert_id_function ? > thd->first_successful_insert_id_in_prev_stmt : > (info.copied ? autoinc_value_of_last_inserted_row : 0)); > - > - ::my_ok(thd, row_count, id, message); > + Trailing whitespace > + /* > + Client expects an EOF/Ok packet If LEX::returning_list is not empty and Trailing whitespace > + if result set meta was sent. See explanation for other variants of INSERT. > + */ > + if (sel_result) > + sel_result->send_eof(); > + else > + ::my_ok(thd, row_count, id, message); > > DBUG_RETURN(false); > } > diff --git a/sql/sql_insert.h b/sql/sql_insert.h > index a37ed1f31e5..314817b53d3 100644 > --- a/sql/sql_insert.h > +++ b/sql/sql_insert.h > @@ -27,11 +27,11 @@ bool mysql_prepare_insert(THD *thd, TABLE_LIST *table_list, TABLE *table, > List<Item> &fields, List_item *values, > List<Item> &update_fields, > List<Item> &update_values, enum_duplicates duplic, > - COND **where, bool select_insert); > + COND **where, bool select_insert, bool with_returning_list); This line exceeds 80 characters. Please wrap it to 80 characters by moving the final function parameter to a new line. > bool mysql_insert(THD *thd,TABLE_LIST *table,List<Item> &fields, > List<List_item> &values, List<Item> &update_fields, > List<Item> &update_values, enum_duplicates flag, > - bool ignore); > + bool ignore, select_result* result); > void upgrade_lock_type_for_insert(THD *thd, thr_lock_type *lock_type, > enum_duplicates duplic, > bool is_multi_insert); > diff --git a/sql/sql_lex.h b/sql/sql_lex.h > index 0e1d17d13f0..00cf3025efe 100644 > --- a/sql/sql_lex.h > +++ b/sql/sql_lex.h > @@ -3091,6 +3092,8 @@ struct LEX: public Query_tables_list > SELECT_LEX *current_select; > /* list of all SELECT_LEX */ > SELECT_LEX *all_selects_list; > + /* List of fields and expression for returning part of insert*/ Add a space before closing the comment. > + List<Item> returning_list; > /* current with clause in parsing if any, otherwise 0*/ > With_clause *curr_with_clause; > /* pointer to the first with clause in the current statement */ > diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc > index 01d0ed1c383..852078b254b 100644 > --- a/sql/sql_parse.cc > +++ b/sql/sql_parse.cc > @@ -4489,6 +4489,7 @@ mysql_execute_command(THD *thd) > case SQLCOM_INSERT: > { > WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE); > + select_result * sel_result = NULL; Tabs instead of spaces. Delete the space after *. Delete the space before =. > DBUG_ASSERT(first_table == all_tables && first_table != 0); > > WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE); > @@ -4509,9 +4510,43 @@ mysql_execute_command(THD *thd) > break; > > MYSQL_INSERT_START(thd->query()); > + > + Protocol* UNINIT_VAR(save_protocol); > + bool replaced_protocol = false; Tabs instead of spaces. Delete space before = > + > + if (!thd->lex->returning_list.is_empty()) > + { > + /*increment the status variable. This is useful for feedback plugin*/ I would remove this comment, the function name is suggestive enough. > + status_var_increment(thd->status_var.feature_insert_returning); > + > + /*This is INSERT ... RETURNING. It will return output to the client */ Space after start of comment. > + if (thd->lex->analyze_stmt) > + { > + /* > + Actually, it is ANALYZE .. INSERT .. RETURNING. We need to produce > + output and then discard it. > + */ Trailign whitespace tab. > + sel_result = new (thd->mem_root) select_send_analyze(thd); Delete space before = > + replaced_protocol = true; Delete space before = > + save_protocol = thd->protocol; Delete space before = > + thd->protocol = new Protocol_discard(thd); Delete space before = > + } > + else > + { > + if (!lex->result && !(sel_result = new (thd->mem_root) select_send(thd))) Delete space before = > + goto error; > + } > + } Tabs instead of spaces for this whole part. > + Trailing whitespaces. > res= mysql_insert(thd, all_tables, lex->field_list, lex->many_values, > lex->update_list, lex->value_list, > - lex->duplicates, lex->ignore); > + lex->duplicates, lex->ignore, lex->result ? lex->result : sel_result); This line overflows the 80 character line rule. Wrap the last parameter to another line. IT also makes things a lot easier to read. > + if (replaced_protocol) > + { Tabs instead of spaces for these 2 lines. > + delete thd->protocol; > + thd->protocol= save_protocol; > + } > + delete sel_result; Tab instead of spaces. > MYSQL_INSERT_DONE(res, (ulong) thd->get_row_count_func()); > /* > If we have inserted into a VIEW, and the base table has > @@ -4547,6 +4582,8 @@ mysql_execute_command(THD *thd) > { > WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_INSERT_REPLACE); > select_insert *sel_result; > + TABLE_LIST *save_first= NULL; Tab instead of space after =. > + select_result* result = NULL; Delete space after =. Move * close to result not to select_result. > bool explain= MY_TEST(lex->describe); > DBUG_ASSERT(first_table == all_tables && first_table != 0); > WSREP_SYNC_WAIT(thd, WSREP_SYNC_WAIT_BEFORE_UPDATE_DELETE); > @@ -4595,6 +4632,34 @@ mysql_execute_command(THD *thd) > Only the INSERT table should be merged. Other will be handled by > select. > */ > + > + Protocol* UNINIT_VAR(save_protocol); > + bool replaced_protocol = false; Tabs instead of spaces. Delete space before =. > + > + if (!thd->lex->returning_list.is_empty()) > + { Tabs instead of spaces. > + /*increment the status variable. This is useful for feedback plugin*/ I would remove this comment, the function name is suggestive enough. > + status_var_increment(thd->status_var.feature_insert_returning); > + > + /* This is INSERT ... RETURNING. It will return output to the client */ > + if (thd->lex->analyze_stmt) > + { > + /* > + Actually, it is ANALYZE .. INSERT .. RETURNING. We need to produce > + output and then discard it. > + */ > + result = new (thd->mem_root) select_send_analyze(thd); > + replaced_protocol = true; > + save_protocol = thd->protocol; > + thd->protocol = new Protocol_discard(thd); > + } > + else > + { > + if (!lex->result && !(result = new (thd->mem_root) select_send(thd))) > + goto error; > + } > + } > + > /* Skip first table, which is the table we are inserting in */ > TABLE_LIST *second_table= first_table->next_local; > /* > @@ -4604,18 +4669,33 @@ mysql_execute_command(THD *thd) > TODO: fix it by removing the front element (restoring of it should > be done properly as well) > */ > + Trailing whitespaces for this line. > + /* > + Also, if items are present in returning_list, then we need those items > + to point to INSERT table during setup_fields() and setup_wild(). But > + it gets masked before that. So we save the values in saved_first, > + saved_table_list and saved_first_name_resolution_context before they are masked. > + We will later swap the saved values with the masked values if returning_list > + is not empty in INSERT...SELECT...RETURNING. This comment text needs to indented 2 more spaces. Remove trailing whitespaces. Also, make sure to keep within 80 characters per line. > + */ > + > + TABLE_LIST *save_first=select_lex->table_list.first; Tabs instead of spaces. Add space after =. > select_lex->table_list.first= second_table; > + select_lex->context.saved_table_list=select_lex->context.table_list; Tabs instead of spaces. Add space after =. > + select_lex->context.saved_name_resolution_table= Tabs instead of spaces. > + select_lex->context.first_name_resolution_table; > select_lex->context.table_list= > select_lex->context.first_name_resolution_table= second_table; > - res= mysql_insert_select_prepare(thd); > - if (!res && (sel_result= new (thd->mem_root) select_insert(thd, > - first_table, > - first_table->table, > - &lex->field_list, > - &lex->update_list, > - &lex->value_list, > - lex->duplicates, > - lex->ignore))) > + res= mysql_insert_select_prepare(thd,lex->result ? lex->result : result); > + if (!res && (sel_result= new (thd->mem_root) select_insert(thd, first_table, > + first_table->table, > + &lex->field_list, > + &lex->update_list, > + &lex->value_list, > + lex->duplicates, > + lex->ignore, > + lex->result ? lex->result : result, > + save_first))) This is ugly. I suggest you keep just new (thd->mem_root) on one line and wrap everything to a new line. Something like: + if (!res && + (sel_result= new (thd->mem_root) + select_insert(thd, first_table, + first_table->table, + &lex->field_list, + &lex->update_list, + &lex->value_list, + lex->duplicates, + lex->ignore, + lex->result ? lex->result : result, + save_first))) > > { > if (lex->analyze_stmt) > ((select_result_interceptor*)sel_result)->disable_my_ok_calls(); > @@ -4630,6 +4710,7 @@ mysql_execute_command(THD *thd) > TODO: this is workaround. right way will be move invalidating in > the unlock procedure. > */ > + Extra line added for no good reason. > if (!res && first_table->lock_type == TL_WRITE_CONCURRENT_INSERT && > thd->lock) > { > @@ -4650,8 +4731,13 @@ mysql_execute_command(THD *thd) > sel_result->abort_result_set(); > } > delete sel_result; > + delete result; > + } > + if (replaced_protocol) > + { > + delete thd->protocol; > + thd->protocol= save_protocol; > } > - No need to delete this empty line. > if (!res && (explain || lex->analyze_stmt)) > res= thd->lex->explain->send_explain(thd); > > diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc > index 8088b6923f2..2f5d0f17451 100644 > --- a/sql/sql_prepare.cc > +++ b/sql/sql_prepare.cc > @@ -1296,7 +1296,7 @@ static bool mysql_test_insert(Prepared_statement *stmt, > > if (mysql_prepare_insert(thd, table_list, table_list->table, > fields, values, update_fields, update_values, > - duplic, &unused_conds, FALSE)) > + duplic, &unused_conds, FALSE,false)) FALSE mixed with false. As this function call uses FALSE, let's use FALSE too. I know, the file is inconsistent, it uses FALSE and false at times. :( Let's do our best here and not make things stranger than they have to be. Space after last comma too. Like so: duplic, &unused_conds, FALSE, FALSE)) > goto error; > > value_count= values->elements; > @@ -2154,7 +2154,7 @@ static int mysql_insert_select_prepare_tester(THD *thd) > thd->lex->first_select_lex()->context.first_name_resolution_table= > second_table; > > - return mysql_insert_select_prepare(thd); > + return mysql_insert_select_prepare(thd,NULL); space after , please. > } > > > diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy > index 183a2504b70..c7706ca3c1f 100644 > --- a/sql/sql_yacc.yy > +++ b/sql/sql_yacc.yy > @@ -1911,11 +1912,14 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); > %type <item_basic_constant> text_literal > > %type <item_list> > - expr_list opt_udf_expr_list udf_expr_list when_list when_list_opt_else > + opt_insert_update Trailing whitespace. > + opt_select_expressions expr_list Trailing whitespace. > + opt_udf_expr_list udf_expr_list when_list when_list_opt_else > ident_list ident_list_arg opt_expr_list > decode_when_list_oracle > execute_using > execute_params > + select_item_list > > %type <sp_cursor_stmt> > sp_cursor_stmt_lex > @@ -9223,6 +9226,7 @@ query_specification_start: > select_item_list > { > Select->parsing_place= NO_MATTER; > + Select->item_list=*($5); Space after =. > } > ; > > @@ -9544,9 +9548,24 @@ opt_lock_wait_timeout_new: > } > ; > > + /* > + Here, we make select_item_list return List<Item> to prevent it from adding > + everything to SELECT_LEX::item_list. If items are already there in the item_list > + then using RETURNING with INSERT...SELECT is not possible because rules occuring > + after insert_values add everything to SELECT_LEX::item_list. > + */ This comment must be wrapped at 80 lines. > + Delete this final line. > select_item_list: > select_item_list ',' select_item > + { > + $1->push_back($3,thd->mem_root); Space after , > + $$=$1; Space after = > + } > | select_item > + { > + if (unlikely(!($$= List<Item>::make(thd->mem_root, $1)))) > + MYSQL_YYABORT; > + } > | '*' > { > Item *item= new (thd->mem_root) > @@ -9554,24 +9573,23 @@ select_item_list: > NULL, NULL, &star_clex_str); > if (unlikely(item == NULL)) > MYSQL_YYABORT; > - if (unlikely(add_item_to_list(thd, item))) > + if (unlikely(!($$= List<Item>::make(thd->mem_root, item)))) > MYSQL_YYABORT; > (thd->lex->current_select->with_wild)++; > + > + (thd->lex->current_select->with_wild)++; This double's with_wild. This is a bug. > } > ; > > select_item: > remember_name select_sublist_qualified_asterisk remember_end > { > - if (unlikely(add_item_to_list(thd, $2))) > - MYSQL_YYABORT; > + $$=$2; > } > | remember_name expr remember_end select_alias > { > DBUG_ASSERT($1 < $3); > - > - if (unlikely(add_item_to_list(thd, $2))) > - MYSQL_YYABORT; > + $$=$2; > if ($4.str) > { > if (unlikely(Lex->sql_command == SQLCOM_CREATE_VIEW && > @@ -13307,13 +13325,15 @@ insert: > Select->set_lock_for_tables($3, true); > Lex->current_select= Lex->first_select_lex(); > } > - insert_field_spec opt_insert_update > - { > + insert_field_spec opt_insert_update opt_select_expressions > + { Trailing whitespace. > + if ($9) > + Lex->returning_list=*($9); > Lex->pop_select(); //main select > if (Lex->check_main_unit_semantics()) > MYSQL_YYABORT; > } > - ; > + ; Trailing whitespace tab. Additionally the semicolon is not Why change this? The semicolon is now indented wrong. Also trailing tab introduced after. > > replace: > REPLACE > @@ -13331,13 +13351,15 @@ replace: > Select->set_lock_for_tables($3, true); > Lex->current_select= Lex->first_select_lex(); > } > - insert_field_spec > + insert_field_spec opt_select_expressions > { > + if ($7) > + Lex->returning_list=*($7); Space after =. > Lex->pop_select(); //main select > if (Lex->check_main_unit_semantics()) > MYSQL_YYABORT; > } > - ; > + ; Why change this? The semicolon is now indented wrong. > > insert_lock_option: > /* empty */ > @@ -13389,8 +13411,8 @@ insert_table: > ; > > insert_field_spec: > - insert_values {} > - | insert_field_list insert_values {} > + insert_values Trailing whitespace. > + | insert_field_list insert_values > | SET > { > LEX *lex=Lex; > @@ -13732,6 +13754,8 @@ single_multi: > { > if ($3) > Select->order_list= *($3); > + if($5) > + Select->item_list=*($5); Space after if. Space after =. > Lex->pop_select(); //main select > } > | table_wild_list > @@ -13764,9 +13788,16 @@ single_multi: > } > ; > > + /* Trailing whitespace. > + Return NULL if the rule is empty else return the list of items Trailing whitespace. > + in the select expression Trailing whitespce. Put a fullstop at the end of the sentence. > + */ Comment should start at the beginning of line, not indented in this case. > opt_select_expressions: > - /* empty */ > - | RETURNING_SYM select_item_list > + /* empty */ {$$=NULL;} Space after =. Space after {. Space after ; > + | RETURNING_SYM select_item_list > + { > + $$=$2; Space after =. > + } > ; > > table_wild_list:

2 1

[Maria-developers] Weekly Report
by Rucha Deodhar 05 Aug '19

05 Aug '19

Report for week 10: Hello! Report for Week 10: Hello! This week I worked on adding system variable feature_insert_returning which is useful for feedback plugin. This variable increments each time INSERT...RETURNING or REPLACE...RETURNING is used. Before beginning, I did some reading about feedback plugin and referred feature_subquery for adding feature_insert_returning. Initially I was facing difficulty in finding the test file that was to be updated. I was searching for test file in sys_vars test suite. Zulip conversation helped me find the right test file and also a missed step for publishing variable for SHOW STATUS. I have made necessary changes in the code and have updated the test and result file. I also found out that the current implementation was not showing the expected output when AUTO_INCREMENT is used and the fields are not given explicitly. On using RETURNING it was showing the value of all the rows in auto_increment field as 0. Fixed that by calling send_data() after write_record(). This also fixed another thing. The previous implementation, for INSERT ... ON DUPLICATE KEY UPDATE...RETURNING was not showing the updated value in the result set. Example: If the table t1 has 1 row: id1=1 and val1='A'. The below statement INSERT INTO t1(id1,val1) VALUES(1,'B') ON DUPLICATE KEY UPDATE val1='C' returning *; was showing id1=1 and val='B' for the result even if it inserts val1='C'. Now it is fixed and is returning val1='C'. I have updated the result file for insert_returning. I also fixed line endings for tests and added more comments to make our implementation more understandable. I github repo is up to date with the latest changes. Regards, Rucha Deodhar.

1 0

Re: [Maria-developers] [Commits] 1e58c579f5b: MDEV-18930: Failed CREATE OR REPLACE TEMPORARY not written into binary log makes data on master and slave diverge
by Sachin Setiya 02 Aug '19

02 Aug '19

Hi Sujatha! On Thu, Aug 1, 2019 at 4:16 PM sujatha <sujatha.sivakumar(a)mariadb.com> wrote: > > revision-id: 1e58c579f5b61644a74045a0470d2c62b343eb75 (mariadb-10.1.41-3-g1e58c579f5b) > parent(s): 0bb8f33b55cc016d0ede86b97990a3ee30dcb069 > author: Sujatha > committer: Sujatha > timestamp: 2019-08-01 16:16:05 +0530 > message: > > MDEV-18930: Failed CREATE OR REPLACE TEMPORARY not written into binary log makes data on master and slave diverge > > Problem: > ======= > Failed CREATE OR REPLACE TEMPORARY TABLE statement which dropped the table but > failed at a later stage of creation of temporary table is not written to > binarylog in row based replication. This causes the slave to diverge. > > Analysis: > ======== > CREATE OR REPLACE statements work as shown below. > > CREATE OR REPLACE TABLE table_name (a int); > is basically the same as: > > DROP TABLE IF EXISTS table_name; > CREATE TABLE table_name (a int); > > Hence every CREATE OR REPLACE TABLE command which dropped the table should be > written to binary log, even when following CREATE TABLE part fails. In order > to achieve this, during the execution of CREATE OR REPLACE command, when a > table is dropped 'thd->log_current_statement' flag is set. When table creation > results in an error within 'mysql_create_table' code, the error handling part > looks for this flag. If it is set the failed CREATE OR REPLACE statement is > written into the binary log inspite of error. This ensure that slave doesn't > diverge from the master. In case of row based replication the error handling > code returns very early, if the table is of type temporary. This is done based > on the assumption that temporary tables are not replicated in row based > replication. > > It fails to handle the cases where a temporary table was created as part of > statement based replication at an earlier stage and the binary log format was > changed to row because of an unsafe statement. In this case when a CREATE OR > REPLACE statement is executed on this temporary table it will dropped but the > query will not be written to binary log. Hence slave diverges. > > Fix: > === > In error handling code check the return status of create table operation. If > it is successful and replication mode is row based and table is of type > temporary then return. Other wise proceed further to the code which checks for > thd->log_current_statement flag and does appropriate logging. > > --- > .../suite/rpl/r/rpl_create_or_replace_fail.result | 18 +++++++ > .../suite/rpl/t/rpl_create_or_replace_fail.test | 56 ++++++++++++++++++++++ > sql/sql_table.cc | 2 +- > 3 files changed, 75 insertions(+), 1 deletion(-) > > diff --git a/mysql-test/suite/rpl/r/rpl_create_or_replace_fail.result b/mysql-test/suite/rpl/r/rpl_create_or_replace_fail.result > new file mode 100644 > index 00000000000..57178f0efbe > --- /dev/null > +++ b/mysql-test/suite/rpl/r/rpl_create_or_replace_fail.result > @@ -0,0 +1,18 @@ > +include/master-slave.inc > +[connection master] > +CREATE TEMPORARY TABLE t1 (a INT NOT NULL); > +LOAD DATA INFILE 'x' INTO TABLE x; > +ERROR 42S02: Table 'test.x' doesn't exist > +CREATE OR REPLACE TEMPORARY TABLE t1 (x INT) PARTITION BY HASH(x); > +ERROR HY000: Cannot create temporary table with partitions > +"************** DROP TEMPORARY TABLE Should be present in Binary log **************" > +include/show_binlog_events.inc > +Log_name Pos Event_type Server_id End_log_pos Info > +master-bin.000001 # Gtid # # GTID #-#-# > +master-bin.000001 # Query # # use `test`; CREATE TEMPORARY TABLE t1 (a INT NOT NULL) > +master-bin.000001 # Gtid # # GTID #-#-# > +master-bin.000001 # Query # # use `test`; CREATE OR REPLACE TEMPORARY TABLE t1 (x INT) PARTITION BY HASH(x) > +CREATE TABLE t1 (b INT); > +INSERT INTO t1 VALUES (NULL); > +DROP TABLE t1; > +include/rpl_end.inc > diff --git a/mysql-test/suite/rpl/t/rpl_create_or_replace_fail.test b/mysql-test/suite/rpl/t/rpl_create_or_replace_fail.test > new file mode 100644 > index 00000000000..e75f34b0b56 > --- /dev/null > +++ b/mysql-test/suite/rpl/t/rpl_create_or_replace_fail.test > @@ -0,0 +1,56 @@ > +# ==== Purpose ==== > +# > +# Test verifies that failed CREATE OR REPLACE TEMPORARY TABLE statement which > +# dropped the table but failed at a later stage of creation of temporary table > +# is written to binarylog in row based replication. > +# > +# ==== Implementation ==== > +# > +# Steps: > +# 0 - Have mixed based replication mode. > +# 1 - Create a temporary table. It will be replicated as mixed replication > +# mode is in use. > +# 2 - Execute an unsafe statement which will switch current statement > +# binlog format to 'ROW'. i.e If binlog_format=MIXED, there are open > +# temporary tables, and an unsafe statement is executed, then subsequent > +# statements are logged in row format. Why not just set binlog format to row ? > +# 3 - Execute a CREATE OR REPLACE TEMPORARY TABLE statement which tries to > +# create partitions on temporary table. Since it is not supported it will > +# fail. > +# 4 - Check the binary log output to ensure that the failed statement is > +# written to the binary log. > +# 5 - Slave should be up and running and in sync with master. > +# > +# ==== References ==== > +# > +# MDEV-18930: Failed CREATE OR REPLACE TEMPORARY not written into binary log > +# makes data on master and slave diverge > +# > + > +--source include/have_partition.inc > +--source include/have_binlog_format_mixed.inc > +--source include/master-slave.inc > + > +CREATE TEMPORARY TABLE t1 (a INT NOT NULL); > + > +# Execute an unsafe statement which switches replication mode internally from > +# "STATEMENT" to "ROW". > +--error ER_NO_SUCH_TABLE > +LOAD DATA INFILE 'x' INTO TABLE x; > + > +--error ER_PARTITION_NO_TEMPORARY > +CREATE OR REPLACE TEMPORARY TABLE t1 (x INT) PARTITION BY HASH(x); > + > +--echo "************** DROP TEMPORARY TABLE Should be present in Binary log **************" > +--source include/show_binlog_events.inc > + > +CREATE TABLE t1 (b INT); > +INSERT INTO t1 VALUES (NULL); > +--sync_slave_with_master > + > +# Cleanup > +--connection master > +DROP TABLE t1; > + > +--source include/rpl_end.inc > + > diff --git a/sql/sql_table.cc b/sql/sql_table.cc > index 656834c7852..fe923d73200 100644 > --- a/sql/sql_table.cc > +++ b/sql/sql_table.cc > @@ -5092,7 +5092,7 @@ bool mysql_create_table(THD *thd, TABLE_LIST *create_table, > > err: > /* In RBR we don't need to log CREATE TEMPORARY TABLE */ > - if (thd->is_current_stmt_binlog_format_row() && create_info->tmp_table()) > + if (!result && thd->is_current_stmt_binlog_format_row() && create_info->tmp_table()) > DBUG_RETURN(result); Can we only binary log only the drop part ? not the create part. > > if (create_info->tmp_table()) > _______________________________________________ > commits mailing list > commits(a)mariadb.org > https://lists.askmonty.org/cgi-bin/mailman/listinfo/commits -- Regards Sachin Setiya Software Engineer at MariaDB

1 0

[Maria-developers] Weekly Report
by Rucha Deodhar 29 Jul '19

29 Jul '19

Report for week 9: Hello! This week I worked on documentation, added some comments to the code and removed some lines of code from parser as it was still making use of SELECT_LEX::item_list after opt_select_expressions. I was referring DELETE...RETURNING test case and noticed case for using stored functions was missing. I added that to INSERT...RETURNING and REPLACE...RETURNING. Also checked if the implementation supports other functions like string functions, date-time functions, control flow functions, numeric function and secondary functions(Since the aggregate functions cannot be used, I wanted to check if these functions can be used) I have made related changes in the documentation and my github repo is up to date with latest changes. Regards, Rucha Deodhar

1 0

[Maria-developers] Windows line endings in MariaDB sources again (
by Otto Kekäläinen 23 Jul '19

23 Jul '19

Hello! While trying to release 10.0.26 for Debian and Ubuntu I noticed git chokes again on Windows line endings. As our .gitattributes states that by default git should clean up the line endings, why do we keep getting new Windows line endings in new code? mariadb-10.0$ cat .gitattributes # Set the default behavior, in case people don't have core.autocrlf set. * text=auto ... storage/connect/mysql-test/connect/std_data/*.txt -text mariadb-10.0.26$ find . | xargs file | grep CRLF ./storage/connect/mysql-test/connect/std_data/boyswin.txt: ASCII text, with CRLF line terminators ./storage/connect/mysql-test/connect/std_data/expenses.txt: ASCII text, with CRLF line terminators ./storage/connect/mysql-test/connect/std_data/emp.txt: ASCII text, with CRLF line terminators ./mysql-test/std_data/loaddata7.dat: ASCII text, with CRLF line terminators ./mysql-test/r/loadxml.result: ASCII text, with CRLF, LF line terminators ./mysql-test/r/perror-win.result: ASCII text, with CRLF, LF line terminators ./mysql-test/r/mysql_binary_mode.result: ASCII English text, with CRLF, LF line terminators ./mysql-test/r/func_regexp_pcre.result: UTF-8 Unicode C++ program text, with CRLF, CR, LF line terminators ./pcre/testdata/grepoutputN: ASCII text, with CRLF, CR, LF line terminators ./pcre/testdata/greppatN4: ASCII text, with CRLF line terminators This leads for me to: dpkg-source: info: building mariadb-10.0 using existing ./mariadb-10.0_10.0.26.orig.tar.gz dpkg-source: info: local changes detected, the modified files are: mariadb-10.0/storage/connect/mysql-test/connect/std_data/boyswin.txt mariadb-10.0/storage/connect/mysql-test/connect/std_data/emp.txt mariadb-10.0/storage/connect/mysql-test/connect/std_data/expenses.txt dpkg-source: error: aborting due to unexpected upstream changes, see /tmp/mariadb-10.0_10.0.26-1.diff.Yzwwhw Which means I have to do extra work with importing the upstream sources.. - Otto

2 4

[Maria-developers] Weekly Report
by Rucha Deodhar 22 Jul '19

22 Jul '19

Report for week 8: Hello! This week I worked on the parser. I removed the hack "swapping lists" we had used earlier. Now the grammar uses values from the bison stack. However, I was making the list rule return the SELECT_LEX :: item_list. This was fixed after the intermediate review. I changed the grammar for INSERT...RETURNING and REPLACE...RETURNING, added test and result files for REPLACE...RETURNING and modified the comments of one test file. I also worked on the intermediate review. Now the parser code doesn't rely on SELECT_LEX :: item_list. In REPLACE...RETURNING a list was declared inside the scope of "if" and using it outside the scope was illegal. So I moved it to the scope outside of if and made the grammar return: &list if select_lex::item_list is not empty else NULL. My github repo is up to date with latest changes. Regards, Rucha Deodhar

1 0

[Maria-developers] INSERT ... RETURNING intermediate review
by Vicențiu Ciorbaru 21 Jul '19

21 Jul '19

Hi Rucha! I've spent quite a bit of time looking at the implementation. It is working and passes almost all test cases, although you forgot to also commit the insert_parser updated result file. Good job so far. I want to give you something to work on and think about for a bit before we do another overall overview of the whole project. Hopefully this will help you across all your coding endeavours, not just now. What I want you to learn is that it is important to think about how your code can and will evolve in the future. Just getting things to work is ok up to a certain point, but not implementing things in a decoupled fashion is what we call "hacks". These hacks are to be avoided as much as possible as it leads to much more difficult work down the line. Let's elaborate on the topic. There are a number of problems with our current code (hacks). I am not referring to yours in particular. All these hacks are a bit difficult to overcome in one go. I do not expect you to fix them yourself, not during this GSoC at least, but it is something to consider and if you are willing to work towards fixing some of them, excellent! Historically our codebase evolved from a simple parser to the "monster" that you see now. Sometimes due to time constraints, things were not implemented in the most extendable way, rather they made use of certain particular constructs, only valid in the contexts they were written in. Keyword here is context, as you'll soon see. Some of this contain things you have interacted with so far in your project. Here is an example that you've had to deal with: A SELECT statement has what we call a SELECT LIST, this list is traditionally stored in SELECT_LEX::item_list. This item_list however is overloaded at times. For example DELETE ... RETURNING uses it to store the expressions part of the RETURNING clause. This overloading generally works ok, unless you run into situations like you did with your INSERT ... RETURNING project. INSERT ... RETURNING clauses already made use of the item_list to store something else. Now you were forced to introduce another list in SELECT_LEX, which we called returning_list. Everything ok so far. But then, you ran into another problem, the bison grammar rules do not put Items in the list you want. They always used item_list, so you could not use your returning_list. So what did we do then? Well, we came up with another hack! We suggested this hack to you because it is something that will work quickly and get you unstuck and that is to use the same grammar rules like before, but swap the item_list with the returning list temporarily, such that the grammar rules will put the right items in the right list. This works, but it masks the underlying problem. The problem is that we have a very rigid implementation for select_item_list grammar rule. What we should do is to make select_item_list grammar return a list using the bison stack. That means no relying on SELECT_LEX::item_list to store our result temporarily. You have done steps towards fixing this, which brings us to where your code's current state. The problem with your implementation is you have not lifted the restriction of the grammar rules making use of SELECT_LIST::item_list. Instead, you have masked it by returning the address of that member on the bison stack. The funny part is that this works, but it still a hack. It is still a hack, because these grammar rules have a side effect of modifying this member behind the scenes. A very, very, very (!) good rule is to consider all side effects as bad, especially now that you are starting out. With experience you might get away with them every now-and-then, but for now avoid them like the plague and try to remove them whenever possible. The current code makes use of a function I really don't like. The inline function add_item_list(). It is one of those hacky functions that strictly relies on context. The context is: take the thd->lex->current_select and put the item passed as a parameter in the item_list. The diff I've attached shows an implementation of how to use the bison stack properly. I took inspiration from the "expr" rule. Unfortunately the hack chain goes deeper, but we need to do baby steps and some things are best left outside your GSoC project, or we risk going down a rabbit whole we might not get out of. One other issue I've observed is with the REPLACE code here: insert_field_spec { if ($6) { List<Item> list; list=*($6); Lex->current_select- >item_list.empty(); $6=&list; } } opt_ select_expressions { if ($8) Lex- >returning_list=*($8); if ($6) Lex->current_select- >item_list=*($6); Lex->pop_select(); //main select if (Lex->check_main_unit_semantics()) MYSQL_YYABORT; } This is really problematic and your probably wrote it because of opt_select_expressions putting stuff into item_list. In the insert_field_spec rule, you receive the address of current_select->item_list as parameter $6. You store the value of this list (basically the pointer to the head element) in a temporary variable that is only in scope during the if statement (!). Attempting to use it after the if statement is over (via the $6 parameter) is illegal! It only works because the compiler did not chose to use that area of memory for something else. Please find a way to store things here properly. After this is fixed, we'll spend the next month cleaning everything up, documenting our code, the added test cases etc. We have an overall working solution, but we need to make it as hack-free as possible, so we can later work on supporting INSERT ... RETURNING in subqueries. Let me know if anything in the email is unclear. It took me a while to write it, trying to explain the reasoning, but I may have missed a few things. :) Vicențiu

2 2

Re: [Maria-developers] MDEV-19429: Wrong query result with EXISTS and LIMIT 0
by Sergey Petrunia 19 Jul '19

19 Jul '19

Hi Sanja, Ok to push after the below input is addressed. > commit ab5fa406b4b314705cb87ffd74111a518b549ff4 > Author: Oleksandr Byelkin <sanja(a)mariadb.com> > Date: Wed Jul 17 12:31:45 2019 +0200 > > MDEV-19429: Wrong query result with EXISTS and LIMIT 0 > > Check EXISTS LIMIT before rewriting. > > diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc > index afc42dc08d5..85d91181337 100644 > --- a/sql/item_subselect.cc > +++ b/sql/item_subselect.cc > @@ -1432,12 +1432,18 @@ void Item_exists_subselect::fix_length_and_dec() > { > DBUG_ENTER("Item_exists_subselect::fix_length_and_dec"); > init_length_and_dec(); > - /* > - We need only 1 row to determine existence (i.e. any EXISTS that is not > - an IN always requires LIMIT 1) > - */ > - thd->change_item_tree(&unit->global_parameters->select_limit, > - new Item_int((int32) 1)); > + // If limit is not set or it is constant more than 1 > + if (!unit->global_parameters->select_limit || > + (unit->global_parameters->select_limit->basic_const_item() && > + unit->global_parameters->select_limit->val_int() > 1)) > + { > + /* > + We need only 1 row to determine existence (i.e. any EXISTS that is not > + an IN always requires LIMIT 1) > + */ > + thd->change_item_tree(&unit->global_parameters->select_limit, > + new Item_int((int32) 1)); Please fix identation ^ > + } > DBUG_PRINT("info", ("Set limit to 1"));a Please move the DBUG_PRINT into the if () {...} . Because right now it will print "set limit to 1" even when it didn't set it. > DBUG_VOID_RETURN; > } I also observe that LIMIT clause is not printed into EXPLAIN EXTENDED output: mysql> explain extended select * from t10 where exists (select * from one_k where a >55 order by a limit 100 offset 50); +------+-------------+-------+------+---------------+------+---------+------+------+----------+-----------------------------+ | id | select_type | table | type | possible_keys | key | key_len | ref | rows | filtered | Extra | +------+-------------+-------+------+---------------+------+---------+------+------+----------+-----------------------------+ | 1 | PRIMARY | t10 | ALL | NULL | NULL | NULL | NULL | 3 | 100.00 | | | 2 | SUBQUERY | one_k | ALL | NULL | NULL | NULL | NULL | 1342 | 100.00 | Using where; Using filesort | +------+-------------+-------+------+---------------+------+---------+------+------+----------+-----------------------------+ 2 rows in set, 1 warning (6.34 sec) mysql> show warnings\G *************************** 1. row *************************** Level: Note Code: 1003 Message: select `test`.`t10`.`a` AS `a` from `test`.`t10` where exists(select 1 from `test`.`one_k` where (`test`.`one_k`.`a` > 55) order by `test`.`one_k`.`a`) 1 row in set (0.00 sec) ^^^ Note the lack of LIMIT above. It's missing only for EXISTS subqueries, for other kinds of subqueries it is there. I guess this is outside of scope of this MDEV and should be filed separately. BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

Re: [Maria-developers] Interaction between rpl_slave_state and rpl_binlog_state
by Kristian Nielsen 18 Jul '19

18 Jul '19

Andrei Elkin <andrei.elkin(a)mariadb.com> writes: > I would also raise another but relevant topic of maintaining > > gtid_"executed"_pos > > which is an union of all GTID executed regardless of their arrival > method. E.g some of foreign (to the recipient server) domains gtid:s may > The master potentially could be (made) interested in such table > should create a replication mode without necessary binlogging on the > server. Sorry, I don't follow. Do you mean to create a new table of GTID positions, which will be updated by all transactions, whether originating locally or replicated by a slave thread? Or do you mean to have the mysql.gtid_slave_pos table updated also by locally originated transactions? Or do you mean to have a new system variable @@gtid_executed_pos which is constructed from the existing binlog and mysql.gtid_slave_pos_table, similar to @@gtid_current_pos, but maybe including more GTIDs somehow? > This is not really something new as it's exactly how mysql implemented > gtid bookkeeping. I think MySQL originally kept track of GTIDs only in the binlog, right? And binlog was required to be enabled for GTID to work? I do remember seeing something that relaxed this requirement, but I haven't followed the details. - Kristian.

2 2

Re: [Maria-developers] [Commits] fb619954867: MDEV-15010: Wrong Seconds_Behind_Master when only starting the SQL_Thread
by andrei.elkin＠pp.inet.fi 16 Jul '19

16 Jul '19

Sujatha, howdy. Thanks for a good piece of work! The fact that it's actually an upstream contribution only pleases more :-), and to that regard we need a commit id to refer to (and to the people behind that work :-)). > revision-id: fb619954867232b3a733213a2703b2a425dff8e9 (mariadb-5.5.64-19-gfb619954867) > parent(s): 7a7d9904e12335ee8b1eea9671138b3c469a3829 > author: Sujatha > committer: Sujatha > timestamp: 2019-06-19 14:48:18 +0530 > message: > > MDEV-15010: Wrong Seconds_Behind_Master when only starting the SQL_Thread > > Problem: > ======= > When I run replication normally, I get a value in "Seconds_Behind_Master" from > "SHOW SLAVE STATUS". However, when I run only the SQL Thread (on local relay > logs that have been downloaded previously), I have "NULL" in > "Seconds_Behind_Master". I would expect to have a numeric value. > > Fix: > === > Implemented following changes. > case 1: "Seconds_Behind_Master" shows 0, when SQL thread is in sync with IO > thread and IO thread is running. > case 2: "Seconds_Behind_Master" reports NULL, when SQL thread is in sync with > IO thread and IO thread is stopped. > case 3: "Seconds_Behind_Master" reports NULL, when SQL thread is stopped while > IO thread is up and running. > case 4: "Seconds_Behind_Master" reports a valid numerical value when IO thread > is stopped and SQL thread is consuming existing relay log. I have a question to the case 4 in the tests actually. > > --- > .../suite/rpl/r/rpl_seconds_behind_master.result | 40 ++++++++++ > .../suite/rpl/t/rpl_seconds_behind_master.test | 92 ++++++++++++++++++++++ > sql/slave.cc | 82 ++++++++++++------- > 3 files changed, 187 insertions(+), 27 deletions(-) > > diff --git a/mysql-test/suite/rpl/r/rpl_seconds_behind_master.result b/mysql-test/suite/rpl/r/rpl_seconds_behind_master.result > new file mode 100644 > index 00000000000..1c0ce7c1b19 > --- /dev/null > +++ b/mysql-test/suite/rpl/r/rpl_seconds_behind_master.result > @@ -0,0 +1,40 @@ > +include/master-slave.inc > +[connection master] > +CREATE TABLE t1 (a INT PRIMARY KEY, b INT); > +****************************************************************** > +* Case1: SQL thread is in sync with IO thread and IO thread is UP > +* Seconds_Behind_Master should be '0'. > +****************************************************************** > +include/assert.inc [Seconds_Behind_Master should be 0] > +****************************************************************** > +* Case2: SQL thread is in sync with IO thread and IO thread is > +* stopped. > +* Seconds_Behind_Master should be 'NULL'. > +****************************************************************** > +include/stop_slave_io.inc > +include/assert.inc [Seconds_Behind_Master should be NULL] > +****************************************************************** > +* Case3: SQL thread is stopped while IO thread is up and running. > +* Seconds_Behind_Master should be 'NULL'. > +****************************************************************** > +START SLAVE IO_THREAD; > +include/wait_for_slave_io_to_start.inc > +include/stop_slave_sql.inc > +include/assert.inc [Seconds_Behind_Master should be NULL] > +****************************************************************** > +* Case4: IO thread is stopped. SQL thread is consuming existing relay > +* log. Hence the Seconds_Behind_Master should have a numeric > +* value > 0 > +****************************************************************** > +INSERT INTO t1 VALUES (1,sleep(2)); > +INSERT INTO t1 VALUES (2,sleep(2)); > +include/sync_slave_io_with_master.inc > +include/stop_slave_io.inc > +START SLAVE SQL_THREAD; > +include/wait_for_slave_sql_to_start.inc > +include/assert.inc [Seconds_Behind_Master should be > 0] > +START SLAVE IO_THREAD; > +include/wait_for_slave_io_to_start.inc > +******* CLEANUP ********* > +DROP TABLE t1; > +include/rpl_end.inc > diff --git a/mysql-test/suite/rpl/t/rpl_seconds_behind_master.test b/mysql-test/suite/rpl/t/rpl_seconds_behind_master.test > new file mode 100644 > index 00000000000..6f9ed2939e1 > --- /dev/null > +++ b/mysql-test/suite/rpl/t/rpl_seconds_behind_master.test > @@ -0,0 +1,92 @@ > +# ==== Purpose ==== > +# > +# Test verifies that Seconds_Behind_Master(SBM) values are as expected in various > +# scenario listed below. > +# > +# case 1: SBM=0 if SQL thread is in sync with IO thread and IO thread is UP > +# case 2: SBM=NULL if SQL thread is in sync with IO thread and IO thread is > +# stopped. > +# case 3: SBM=NULL if SQL thread is stopped while IO thread is up and running. > +# case 4: SBM > 0 and SBM != NULL if IO thread is stopped. SQL thread is > +# consuming existing relay log. > +# > +# ==== Implementation ==== > +# > +# Steps: > +# 1 - Create a table and sync it with slave and verify SBM value. > +# 2 - Now stop IO thread and verify SBM > +# 3 - Start IO thread, Stop SQL thread and verify SBM > +# 4 - Execute slow INSERT statements with SLEEP(2) on master. Ensure that > +# the IO thread has read these INSERT statements and then bring it down. > +# 5 - Start the SQL thread and verify that while it is trying to consume > +# these slow INSERT statements in relay log the SBM value is > 0. > +# > +# ==== References ==== > +# > +# MDEV-15010: Wrong Seconds_Behind_Master when only starting the SQL_Thread. > + > +--source include/have_binlog_format_statement.inc > +--source include/master-slave.inc > + > +--disable_query_log > +CALL mtr.add_suppression("Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT"); > +--enable_query_log > + > +CREATE TABLE t1 (a INT PRIMARY KEY, b INT); > +--sync_slave_with_master > + > +-- echo ****************************************************************** > +-- echo * Case1: SQL thread is in sync with IO thread and IO thread is UP > +-- echo * Seconds_Behind_Master should be '0'. > +-- echo ****************************************************************** > +--let $assert_cond= [SHOW SLAVE STATUS, Seconds_Behind_Master, 1] = 0 > +--let $assert_text= Seconds_Behind_Master should be 0 > +--source include/assert.inc > + > +-- echo ****************************************************************** > +-- echo * Case2: SQL thread is in sync with IO thread and IO thread is > +-- echo * stopped. > +-- echo * Seconds_Behind_Master should be 'NULL'. > +-- echo ****************************************************************** > +--source include/stop_slave_io.inc > +--let $assert_cond= "[SHOW SLAVE STATUS, Seconds_Behind_Master, 1]" = "NULL" > +--let $assert_text= Seconds_Behind_Master should be NULL > +--source include/assert.inc > + > +-- echo ****************************************************************** > +-- echo * Case3: SQL thread is stopped while IO thread is up and running. > +-- echo * Seconds_Behind_Master should be 'NULL'. > +-- echo ****************************************************************** > +START SLAVE IO_THREAD; > +--source include/wait_for_slave_io_to_start.inc > +--source include/stop_slave_sql.inc > +--let $assert_cond= "[SHOW SLAVE STATUS, Seconds_Behind_Master, 1]" = "NULL" > +--let $assert_text= Seconds_Behind_Master should be NULL > +--source include/assert.inc > + > +-- echo ****************************************************************** > +-- echo * Case4: IO thread is stopped. SQL thread is consuming existing relay > +-- echo * log. Hence the Seconds_Behind_Master should have a numeric > +-- echo * value > 0 > +-- echo ****************************************************************** > +--connection master > +--disable_warnings > +INSERT INTO t1 VALUES (1,sleep(2)); > +INSERT INTO t1 VALUES (2,sleep(2)); > +--enable_warnings > +--source include/sync_slave_io_with_master.inc > +--source include/stop_slave_io.inc > + > +START SLAVE SQL_THREAD; > +--source include/wait_for_slave_sql_to_start.inc There's a theoretical (and even though I think it would hardly ever be practical) chance for the SQL thread to catch up while for some concurrency reason the waiting would take longer that it's naturally expected. How about to + LOCK TABLES t1 WRITE before START SLAVE SQL_THREAD and unlock > +--let $assert_cond= [SHOW SLAVE STATUS, Seconds_Behind_Master, 1] > 0 > +--let $assert_text= Seconds_Behind_Master should be > 0 > +--source include/assert.inc after the assert check? This measure would speak rather explicitly that the check is done just before any first event has been processed by the SQL thread, and as you prepare 2, then no synchronization is guaranteed. Cheers, Andrei > +START SLAVE IO_THREAD; > +--source include/wait_for_slave_io_to_start.inc > + > +--echo ******* CLEANUP ********* > +--connection master > +DROP TABLE t1; > + > +--source include/rpl_end.inc > diff --git a/sql/slave.cc b/sql/slave.cc > index a8946c69d18..7b4b7bd857e 100644 > --- a/sql/slave.cc > +++ b/sql/slave.cc > @@ -2192,38 +2192,66 @@ bool show_master_info(THD* thd, Master_info* mi) > protocol->store(mi->ssl_cert, &my_charset_bin); > protocol->store(mi->ssl_cipher, &my_charset_bin); > protocol->store(mi->ssl_key, &my_charset_bin); > - > /* > - Seconds_Behind_Master: if SQL thread is running and I/O thread is > - connected, we can compute it otherwise show NULL (i.e. unknown). > + The pseudo code to compute Seconds_Behind_Master: > + if (SQL thread is running) > + { > + if (SQL thread processed all the available relay log) > + { > + if (IO thread is running) > + print 0; > + else > + print NULL; > + } > + else > + compute Seconds_Behind_Master; > + } > + else > + print NULL; > */ > - if ((mi->slave_running == MYSQL_SLAVE_RUN_CONNECT) && > - mi->rli.slave_running) > + if (mi->rli.slave_running) > { > - long time_diff= ((long)(time(0) - mi->rli.last_master_timestamp) > - - mi->clock_diff_with_master); > /* > - Apparently on some systems time_diff can be <0. Here are possible > - reasons related to MySQL: > - - the master is itself a slave of another master whose time is ahead. > - - somebody used an explicit SET TIMESTAMP on the master. > - Possible reason related to granularity-to-second of time functions > - (nothing to do with MySQL), which can explain a value of -1: > - assume the master's and slave's time are perfectly synchronized, and > - that at slave's connection time, when the master's timestamp is read, > - it is at the very end of second 1, and (a very short time later) when > - the slave's timestamp is read it is at the very beginning of second > - 2. Then the recorded value for master is 1 and the recorded value for > - slave is 2. At SHOW SLAVE STATUS time, assume that the difference > - between timestamp of slave and rli->last_master_timestamp is 0 > - (i.e. they are in the same second), then we get 0-(2-1)=-1 as a result. > - This confuses users, so we don't go below 0: hence the max(). > - > - last_master_timestamp == 0 (an "impossible" timestamp 1970) is a > - special marker to say "consider we have caught up". > + Check if SQL thread is at the end of relay log > + Checking should be done using two conditions > + condition1: compare the log positions and > + condition2: compare the file names (to handle rotation case) > */ > - protocol->store((longlong)(mi->rli.last_master_timestamp ? > - max(0, time_diff) : 0)); > + if ((mi->master_log_pos == mi->rli.group_master_log_pos) && > + (!strcmp(mi->master_log_name, mi->rli.group_master_log_name))) > + { > + if (mi->slave_running == MYSQL_SLAVE_RUN_CONNECT) > + protocol->store(0LL); > + else > + protocol->store_null(); > + } > + else > + { > + long time_diff = ((long)(time(0) - mi->rli.last_master_timestamp) > + - mi->clock_diff_with_master); > + /* > + Apparently on some systems time_diff can be <0. Here are possible > + reasons related to MySQL: > + - the master is itself a slave of another master whose time is ahead. > + - somebody used an explicit SET TIMESTAMP on the master. > + Possible reason related to granularity-to-second of time functions > + (nothing to do with MySQL), which can explain a value of -1: > + assume the master's and slave's time are perfectly synchronized, and > + that at slave's connection time, when the master's timestamp is read, > + it is at the very end of second 1, and (a very short time later) when > + the slave's timestamp is read it is at the very beginning of second > + 2. Then the recorded value for master is 1 and the recorded value for > + slave is 2. At SHOW SLAVE STATUS time, assume that the difference > + between timestamp of slave and rli->last_master_timestamp is 0 > + (i.e. they are in the same second), then we get 0-(2-1)=-1 as a result. > + This confuses users, so we don't go below 0: hence the max(). > + > + last_master_timestamp == 0 (an "impossible" timestamp 1970) is a > + special marker to say "consider we have caught up". > + */ > + protocol->store((longlong)(mi->rli.last_master_timestamp ? > + max(0, time_diff) : 0)); > + } > } > else > { > _______________________________________________ > commits mailing list > commits(a)mariadb.org > https://lists.askmonty.org/cgi-bin/mailman/listinfo/commits

1 0

[Maria-developers] MDEV-8306: more input
by Sergey Petrunia 16 Jul '19

16 Jul '19

Hi Varun, Please find more input, I'm commenting on the second patch (the review is probably incomplete, I've made just one pass so might have missed something) > diff -urpN '--exclude=.*' 10.5-mdev8306-orig/sql/opt_subselect.cc 10.5-mdev8306/sql/opt_subselect.cc > --- 10.5-mdev8306-orig/sql/opt_subselect.cc 2019-07-10 11:30:41.151696943 +0300 > +++ 10.5-mdev8306/sql/opt_subselect.cc 2019-07-16 13:07:13.948467315 +0300 > @@ -5517,6 +5517,31 @@ enum_nested_loop_state join_tab_executio > sjm->materialized= TRUE; > } > } > + /* > + This should be the place to add the sub_select call for the > + prefix of the join order > + */ > + > + else if (tab->is_order_nest) > + { > + enum_nested_loop_state rc; > + JOIN *join= tab->join; > + NEST_INFO *nest_info= join->order_nest_info; > + > + if (!nest_info->materialized) > + { > + JOIN_TAB *join_tab= join->join_tab + join->const_tables; > + JOIN_TAB *save_return_tab= join->return_tab; > + if ((rc= sub_select(join, join_tab, FALSE)) < 0 || > + (rc= sub_select(join, join_tab, TRUE)) < 0) > + { > + join->return_tab= save_return_tab; > + DBUG_RETURN(rc); > + } > + join->return_tab= save_return_tab; > + nest_info->materialized= TRUE; > + } Looks a bit repetitive with the SJM nest code above, but this is not a big deal. A question: if the JOIN_TABs for the tables inside the sort nest are still in the join->join_tab array, where is the code to "jump over" them in regular join execution? > + } > > DBUG_RETURN(NESTED_LOOP_OK); > } ... > diff -urpN '--exclude=.*' 10.5-mdev8306-orig/sql/sql_class.h 10.5-mdev8306/sql/sql_class.h > --- 10.5-mdev8306-orig/sql/sql_class.h 2019-07-16 13:10:05.448491384 +0300 > +++ 10.5-mdev8306/sql/sql_class.h 2019-07-16 13:07:13.948467315 +0300 > @@ -6037,6 +6037,24 @@ public: > Copy_field *copy_field; /* Needed for SJ_Materialization scan */ > }; > > +class NEST_INFO : public Sql_alloc > +{ > +public: > + ~NEST_INFO() > + { > + nest_tab= NULL; > + table= NULL; > + } What is this for? Object members should not be used after destructor is called. If you're concerned about accidental access, please use TRASH_XXX macros to fill the variables with "already-freed" marker. > + TMP_TABLE_PARAM tmp_table_param; > + List<Item> nest_base_table_cols; > + List<Item> nest_temp_table_cols; > + TABLE *table; > + st_join_table *nest_tab; > + uint n_tables; > + bool materialized; /* TRUE <=> materialization already performed */ > + table_map nest_tables_map; > +}; > + > > /* Structs used when sorting */ > struct SORT_FIELD_ATTR ... > diff -urpN '--exclude=.*' 10.5-mdev8306-orig/sql/sql_select.cc 10.5-mdev8306/sql/sql_select.cc > --- 10.5-mdev8306-orig/sql/sql_select.cc 2019-07-16 13:10:05.452491478 +0300 > +++ 10.5-mdev8306/sql/sql_select.cc 2019-07-16 13:07:13.952467409 +0300 > return retval; > } > > +void substitute_base_to_nest_items(JOIN *join) > +{ > + NEST_INFO *order_nest_info= join->order_nest_info; > + if (!order_nest_info) > + return; > + REPLACE_NEST_FIELD_ARG arg= {join}; > + join->conds= join->conds->transform(join->thd, &Item::replace_with_nest_items, > + (uchar *) &arg); > + > + List_iterator<Item> it(join->fields_list); > + Item *item, *new_item; > + uint i=0; > + while ((item= it++)) > + { > + if ((new_item= item->transform(join->thd, > + &Item::replace_with_nest_items, > + (uchar *) &arg)) != item) > + it.replace(new_item); > + } Ok, so you are processing the select list, and the WHERE condition. What about the ON expressions, and the right sides of ref access? Also, how do we solve the issue of the same Item_field object being used in two conditions at once: - one inside the sort nest - one outside the sort nest (remember the example with " (inner_tbl_cond1 AND outer_tbl_con1) OR (inner_tbl_cond2 AND outer_tbl_cond2) " > +} > + > > /** ... > @@ -12136,6 +12314,35 @@ end_sj_materialize(JOIN *join, JOIN_TAB > } > > > +enum_nested_loop_state > +end_nest_materialization(JOIN *join, JOIN_TAB *join_tab, bool end_of_records) > +{ > + int error; > + THD *thd= join->thd; > + NEST_INFO *nest_info= join->order_nest_info; > + DBUG_ENTER("end_sj_materialize"); Please change to match the function name. > + if (!end_of_records) > + { > + TABLE *table= nest_info->table; > + fill_record(thd, table, table->field, > + nest_info->nest_base_table_cols, TRUE, FALSE); > + > + if (unlikely(thd->is_error())) > + DBUG_RETURN(NESTED_LOOP_ERROR); /* purecov: inspected */ > + if (unlikely((error= table->file->ha_write_tmp_row(table->record[0])))) > + { > + /* create_myisam_from_heap will generate error if needed */ > + if (table->file->is_fatal_error(error, HA_CHECK_DUP) && Is this check necessary? The table doesn't have any unique keys, why do we ignore duplicates? > + create_internal_tmp_table_from_heap(thd, table, > + nest_info->tmp_table_param.start_recinfo, > + &nest_info->tmp_table_param.recinfo, > + error, 1, NULL)) > + DBUG_RETURN(NESTED_LOOP_ERROR); /* purecov: inspected */ > + } > + } > + DBUG_RETURN(NESTED_LOOP_OK); > +} > + > /* > Check whether a join buffer can be used to join the specified table > BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

Re: [Maria-developers] d7b274fa257: MDEV-16932: ASAN heap-use-after-free in my_charlen_utf8 / my_well_formed_char_length_utf8 on 2nd execution of SP with ALTER trying to add bad CHECK
by Sergei Golubchik 16 Jul '19

16 Jul '19

Hi, Oleksandr! A couple of minor comments, see below On Jul 15, Oleksandr Byelkin wrote: > revision-id: d7b274fa257 (mariadb-10.2.25-63-gd7b274fa257) > parent(s): 64900e3d7c3 > author: Oleksandr Byelkin <sanja(a)mariadb.com> > committer: Oleksandr Byelkin <sanja(a)mariadb.com> > timestamp: 2019-07-11 13:29:51 +0200 > message: > > MDEV-16932: ASAN heap-use-after-free in my_charlen_utf8 / my_well_formed_char_length_utf8 on 2nd execution of SP with ALTER trying to add bad CHECK > > Make automatic name generation during execution (not prepare). > > Check result of memory allocation operation. > > diff --git a/sql/sql_table.cc b/sql/sql_table.cc > index 043cfaaaaaa..636fb9f427c 100644 > --- a/sql/sql_table.cc > +++ b/sql/sql_table.cc > @@ -6205,6 +6210,10 @@ handle_if_exists_options(THD *thd, TABLE *table, Alter_info *alter_info) > Virtual_column_info *check; > TABLE_SHARE *share= table->s; > uint c; > + > + if (fix_constraints_names(thd, &alter_info->check_constraint_list)) > + DBUG_RETURN(TRUE); It's not very logical, I think, to generate constraint names in handle_if_exists_options(), I'd rather move it out and put directly in mysql_alter_table(). > + > while ((check=it++)) > { > if (!(check->flags & Alter_info::CHECK_CONSTRAINT_IF_NOT_EXISTS) && > @@ -6232,10 +6241,44 @@ handle_if_exists_options(THD *thd, TABLE *table, Alter_info *alter_info) > } > } > > - DBUG_VOID_RETURN; > + DBUG_RETURN(FALSE); > } > > > +static bool fix_constraints_names(THD *thd, List<Virtual_column_info> > + *check_constraint_list) > +{ > + List_iterator<Virtual_column_info> it((*check_constraint_list)); > + Virtual_column_info *check; > + uint nr= 1; > + DBUG_ENTER("fix_constraints_names"); > + if (!check_constraint_list) > + DBUG_RETURN(FALSE); > + // Prevent accessing freed memory during generating unique names > + while ((check=it++)) > + { > + if (check->automatic_name) > + { > + check->name.str= NULL; > + check->name.length= 0; > + } > + } > + it.rewind(); > + // Grnerate unique names if needed typo > + while ((check=it++)) > + { > + if (!check->name.length) > + { > + check->automatic_name= TRUE; > + if (make_unique_constraint_name(thd, &check->name, > + check_constraint_list, > + &nr)) > + DBUG_RETURN(TRUE); > + } > + } > + DBUG_RETURN(FALSE); I would remove the first while() at all and just do the second one till the end: bool ret= FALSE; while ((check=it++)) if (check->automatic_name) ret |= make_unique_constraint_name(...); return ret; Because thd->strmake() basically never fails, it doesn't make much sense to optimize for a case when it does. This code above assumes you set automatic_name=TRUE in mysql_prepare_create_table(). > +} > + > /** > Get Create_field object for newly created table by field index. Regards, Sergei

2 1

Re: [Maria-developers] Interaction between rpl_slave_state and rpl_binlog_state
by Kristian Nielsen 15 Jul '19

15 Jul '19

Hi Andrei! The @@gtid_current_pos exists for one sole purpose. This is to let the user promote a slave as the new master and attach the old master as a slave to the new master. By using master_use_gtid=current_pos, the exact same command can be used to attach a slave to the new master, regardless of whether that slave was previously a slave or a master: CHANGE MASTER TO master_host=new_promoted_master If not using gtid_current_pos (ie master_use_gtid=slave_pos), then to let the old master become a slave of the new master, the old master's position must explicitly be set: SET GLOBAL gtid_slave_pos=@@gtid_binlog_pos This is because for efficiency reasons, the master doesn't update the mysql.gtid_slave_pos in each commit. So now we can see why only GTIDs with the servers own server_id should contribute to @@gtid_current_pos. If a GTID was replicated from another server, that GTID will appear in the @@gtid_slave_pos. If the GTID originated on this server, it will appear in @@gtid_binlog_pos. The @@gtid_current_pos is the @@gtid_slave_pos extended with GTIDs originating on this server, hence only GTIDs with our own server id. Normally, every GTID in the binlog with a different server id than our own will already be in the @@gtid_slave_pos as well - since it originated on another server and was replicated to this server. Thus, in the normal case, where user did not play tricks with the binlog and slave state, extending the @@gtid_current_pos as suggested in MDEV-18404 has no effect - the GTIDs are already in the @@gtid_slave_pos, so @@gtid_current_pos is unaffected. And in case the user deliberately modified the state, it should be up to the user to decide what goes into @@gtid_slave_state and @@gtid_binlog_state. For example, the MDEV-18404 change would make it impossible on a server to remove a replicated GTID from the @@gtid_current_pos if --log-slave-updates (without the drastic RESET MASTER). Another problem is that the server cannot reliably compare GTIDs with distinct server ids to decide which one is the most recent. There is no guarantee that sequence numbers are monotonic across different server ids. Thus the MDEV-18404 method could create completely invalid positions in some setups where @@gtid_strict_mode=0 and replication domains are not strictly maintained. I don't see the value in MDEV-18404. If the user is updating @@gtid_binlog_state (itself a very drastic operation), and wants a specific GTID to go into the slave position - just update the @@gtid_slave_pos with the desired GTID, don't leave the server with an inconsistent replication state. And finally, let me reiterate: I consider the @@gtid_current_pos a design mistake. Better to just transfer the @@gtid_binlog_pos to the @@gtid_slave_pos _only_ at the point where an old master is turned into a slave. This can be done manually already, and it would be simple to implement automatic support for this with an extra option for CHANGE MASTER. Hope this helps, - Kristian. Andrei Elkin <andrei.elkin(a)mariadb.com> writes: > Kristian, howdy! > > I thought it's good to refer to a previous discussion that touched > 'current_pos' feature now when we need to resolve MDEV-18404 which I > look at under a specific angle, narrated here. > >> Finally, experience has shown that a _lot_ of users get problems when >> locally done transactions on a slave influence the slave's GTID position. In >> retrospect, I have realised that CHANGE MASTER TO >> master_use_gtid=current_pos was a mistake, only slave_pos should be used. >> Similarly, if a local transaction in a slave's binlog can cause transactions >> from the master to be silently ignored, it will cause a lot of grief for >> users. > > > To the definition of > > gtid_current_pos > > the docs https://mariadb.com/kb/en/library/gtid/#gtid_current_pos first introduce an intuition > with > > This variable is the GTID of the last change to the database for > each replication domain. Such changes can either be master events > (ie. local changes made by user or application), > or replicated events originating from another master server. > > and then define the variable essentially as an union of two sets: > > For each replication domain, if the server ID of the corresponding > GTID in @@gtid_binlog_pos is equal to the servers own server_id, and > the sequence number is higher than the corresponding GTID in > @@gtid_slave_pos, then the GTID from @@gtid_binlog_pos will be > used. Otherwise the GTID from @@gtid_slave_pos will be used for that > domain. > > The union operation is on a *subset* of @@gtid_binlog_pos (call it B') > and the whole @@gtid_slave_pos (S): > > C = B' \cup S > > where C @@gtid_current_pos. > The @@gtid_binlog_pos subset > consists of GTID:s originated by @@global.server_id (that is foreign > ones excluded). > > Notice that makes the union non symmetric as binary operator. A foreign > GTID feels perfectly fine in S and gets into the union result. > > The following use case > > --connection master > CREATE TABLE t (a INT); > RESET MASTER; > SELECT @@global.server_id; > # => 1 > SET @@server_id= 3; > INSERT INTO t SET a=3; > select @@global.gtid_binlog_state = '0-3-1'; > # => 1 > > technically it's a 'local changes', but when S (@@global.gtid_slave_pos) is empty > > SELECT @@global_current_pos = ''; > # => 1 > > This result is another way to read MDEV-18404. > > After thinking over the restricted definition of @@gtid_current_pos I could not arrive at > relaxing it to account the whole @@gtid_binlog_set which fixes everything. > I wonder what do think about this case, have I missed any use case that might have forced > to the definition be as it is? > > Below is a straightforward patch that sustains the current mtr rpl > suites, expect rpl_gtid_startpos (that exactly relies at one point on > the empty @@gtid_current_pos due to the union specifics, I am leaving > out changes to the test at this point). > > Cheers, > > Andrei > > > diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc > index 752b4172b6e..1f9802bd8f8 100644 > --- a/sql/rpl_gtid.cc > +++ b/sql/rpl_gtid.cc > @@ -845,8 +845,7 @@ rpl_slave_state::iterate(int (*cb)(rpl_gtid *, void *), void *data, > my_hash_init(&gtid_hash, &my_charset_bin, 32, offsetof(rpl_gtid, domain_id), > sizeof(uint32), NULL, NULL, HASH_UNIQUE); > for (i= 0; i < num_extra; ++i) > - if (extra_gtids[i].server_id == global_system_variables.server_id && > - my_hash_insert(&gtid_hash, (uchar *)(&extra_gtids[i]))) > + if (my_hash_insert(&gtid_hash, (uchar *)(&extra_gtids[i]))) > goto err; > > mysql_mutex_lock(&LOCK_slave_state);

1 0

Re: [Maria-developers] [Commits] 7cabdc461b2: MDEV-6860 Parallel async replication hangs on a Galera node
by Kristian Nielsen 15 Jul '19

15 Jul '19

sachin.setiya(a)mariadb.com writes: > revision-id: 7cabdc461b24fdebe599799d7964efa4b53815e3 (mariadb-10.1.39-91-g7cabdc461b2) > > MDEV-6860 Parallel async replication hangs on a Galera node > > Wait for previous commit beore preparing next transation for galera > diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc > index 8fef2d66635..7d38c36b840 100644 > --- a/sql/rpl_parallel.cc > +++ b/sql/rpl_parallel.cc > @@ -1181,7 +1181,7 @@ handle_rpl_parallel_thread(void *arg) > before, then wait now for the prior transaction to complete its > commit. > */ > - if (rgi->speculation == rpl_group_info::SPECULATE_WAIT && > + if ((rgi->speculation == rpl_group_info::SPECULATE_WAIT || WSREP_ON) && > (err= thd->wait_for_prior_commit())) Ouch! That's killing _all_ parallel replication when WSREP_ON :-/ Do you really need to do this? It seems quite a restriction for replicating to Galera if parallel replication is not allowed. (I wonder if this isn't just another symptom of the underlying problem that Galera has never been integrated properly into MariaDB and the group commit algorithm / transaction master?). But if the goal is to disable parallel replication in Galera, then you shouldn't do this, it will just confuse/disappoint users, and it will be slower than just using single-threaded replication. Instead, give an error if parallel replication and Galera is enabled at the same time, so users will know of the restriction. - Kristian.

2 2

[Maria-developers] Weekly Report
by Rucha Deodhar 15 Jul '19

15 Jul '19

Report for week 7: Hello! This week I worked on the initial review for INSERT...RETURNING. I removed some lines from my code, used select_result* instead of using with_returning_list, improved formatting, restored the structure of SQL_I_List by removing *saved_first, moved returning_list to LEX so that current_select and lex->first_select_lex() is not used, fixed the logic of my_ok() and send_eof(), and made some part more readable. Now INSERT...IGNORE is working too. For tests, I merged some test cases, removed the other ones and added some cases for existing tests. I also read about bison and how to use its stack to return items so that it can be used instead of list swapping. As suggested, I am making the list rule return the List<Item> as a return value, then the rule that uses select_item_list uses something like this: lex->returning=$n I have started implementing it but getting errors during build. Initially I was getting error: opt_select_expressions has no declared type when I tried making the entire rule return item_list. To fix that I added opt_select_expressions to %type <item_list> I am still trying to fix other errors. Could you give me feedback whether I am moving along the right path, have I missed something? opt_select_expressions: /*empty*/ {$$=NULL} | RETURNING select_item_list { //Make this entire rule return item_list something like: $$=lex->current_select->item_list } ; Insert parser: insert_field_spec opt_insert_update opt_select_expressions { something like if ($3) Lex->returning_list= $3; Lex->pop_select(); //main select if (Lex->check_main_unit_semantics()) MYSQL_YYABORT; } ; Regards, Rucha Deodhar <https://www.avast.com/en-in/recommend?utm_medium=email&utm_source=link&utm_…> I’m protected online with Avast Free Antivirus. Get it here — it’s free forever. <https://www.avast.com/en-in/recommend?utm_medium=email&utm_source=link&utm_…> <#DAB4FAD8-2DD7-40BB-A1B8-4E2AA1F9FDF2>

1 0

[Maria-developers] MDEV-8306: Some input on the code so far
by Sergey Petrunia 10 Jul '19

10 Jul '19

Hi Varun, Please find below some input on the current patch: > diff -urpN '--exclude=.*' 10.5-mdev8306-orig/sql/sql_class.h 10.5-mdev8306/sql/sql_class.h > --- 10.5-mdev8306-orig/sql/sql_class.h 2019-07-10 11:34:00.256256952 +0300 > +++ 10.5-mdev8306/sql/sql_class.h 2019-07-10 11:34:15.436604510 +0300 > @@ -6037,6 +6037,15 @@ public: > Copy_field *copy_field; /* Needed for SJ_Materialization scan */ > }; > > +class NEST_INFO : public Sql_alloc Needs better naming, there are all kinds of nests. Making the first attempt: SORT_NEST_INFO ? > +{ > +public: > + TMP_TABLE_PARAM tmp_table_param; > + List<Item> nest_table_cols; > + TABLE *table; > + uint n_tables; > +}; > + > > /* Structs used when sorting */ > struct SORT_FIELD_ATTR > diff -urpN '--exclude=.*' 10.5-mdev8306-orig/sql/sql_select.cc 10.5-mdev8306/sql/sql_select.cc > --- 10.5-mdev8306-orig/sql/sql_select.cc 2019-07-10 11:34:00.260257044 +0300 > +++ 10.5-mdev8306/sql/sql_select.cc 2019-07-10 11:34:15.448604784 +0300 > @@ -8135,6 +8143,32 @@ choose_plan(JOIN *join, table_map join_t > use_cond_selectivity)) > DBUG_RETURN(TRUE); > } > + trace_plan.end(); > + > + for (uint tablenr=0;tablenr < join->table_count;tablenr++) > + { > + POSITION *pos= &join->best_positions[tablenr]; > + join->order_nest_tables|= pos->table->table->map; > + if (pos->ordering_achieved) > + { > + join->order_nest= TRUE; > + break; > + } > + } The above looks like query plan refinement action, very close to what is done in get_best_combination. Can this code be moved into that function? > + > + if (join->order_nest && unlikely(thd->trace_started())) > + { > + Json_writer_array trace_order_nest(thd, "order_nest"); > + > + for (uint tablenr=0;tablenr < join->table_count;tablenr++) > + { > + POSITION *pos= &join->best_positions[tablenr]; > + trace_order_nest.add_table_name(pos->table); > + if (pos->ordering_achieved) > + break; > + } > + } > + > > /* > Store the cost of this query into a user variable ... > @@ -14041,6 +14186,132 @@ ORDER *simple_remove_const(ORDER *order, > } > > > +/* > + This function basically tries to propgate all the multiple equalites > + for the Field items, so that one can use them to generate QEP that would > + also take into consideration equality propagation > +*/ > +void propagate_equal_field_for_orderby(JOIN *join, ORDER *first_order) > +{ > + ORDER *order; > + table_map not_const_tables= ~join->const_table_map; > + for (order= first_order; order; order= order->next) > + { > + table_map order_tables=order->item[0]->used_tables(); > + if (!(order_tables & not_const_tables)) > + continue; Is the above necessary? I think we have an optimization which removes constant element from ORDER BY list. (Perhaps, that optimization is invoked after the join optimizer. If that is the case, it should be done before the join optimizer) > + else > + { > + if (optimizer_flag(join->thd, OPTIMIZER_SWITCH_ORDERBY_EQ_PROP) && > + order->item[0]->real_item()->type() == Item::FIELD_ITEM && > + join->cond_equal) > + { > + Item *item= order->item[0]; > + /* > + TODO: equality substitution in the context of ORDER BY is > + sometimes allowed when it is not allowed in the general case. > + We make the below call for its side effect: it will locate the > + multiple equality the item belongs to and set item->item_equal > + accordingly. > + */ > + (void)item->propagate_equal_fields(join->thd, > + Value_source:: > + Context_identity(), > + join->cond_equal); > + } > + } > + } > +} > + > +/* > + This function basicall checks if by considering the current join_tab > + would we be able to achieve the ordering > +*/ > + > +bool check_join_prefix_contains_ordering(JOIN *join, JOIN_TAB *tab, > + table_map previous_tables) > +{ > + ORDER *order; > + table_map not_const_tables= ~join->const_table_map; > + for (order= join->order; order; order= order->next) > + { > + table_map order_tables=order->item[0]->used_tables(); > + if (!(order_tables & not_const_tables)) > + continue; Same question as above - at this stage ORDER BY list should not have const elements. > + else > + { > + Item_equal *item_eq= order->item[0]->get_item_equal(); > + if (((previous_tables | tab->table->map) & order_tables) == order_tables || > + (item_eq && (item_eq->used_tables() & (previous_tables | tab->table->map)))) > + continue; > + else > + return FALSE; > + } What about handling for non-const items that are not members of Item_equal? (e.g. ORDER BY left(table.col, 3)) ?) > + } > + return TRUE; > +} > + > + > +bool setup_order_nest(JOIN *join, JOIN_TAB *tab) (Same question about how to call the nest. I would call it "sort nest") > +{ > + if (!tab->is_order_nest) > + return FALSE; > + THD *thd= join->thd; > + Field_iterator_table field_iterator; > + > + JOIN_TAB *start_tab= join->join_tab+join->const_tables, *j; > + NEST_INFO* order_nest_info= join->order_nest_info; > + > + /* This needs to be added to JOIN structure, looks the best option or we > + can have a seperate struture NEST_INFO to hold it > + */ > + > + for (j= start_tab; j < tab; j++) > + { > + TABLE *table= start_tab->table; > + field_iterator.set_table(table); > + for (; !field_iterator.end_of_fields(); field_iterator.next()) > + { > + Field *field= field_iterator.field(); > + if (!bitmap_is_set(table->read_set, field->field_index)) > + continue; > + Item *item; > + if (!(item= field_iterator.create_item(thd))) > + return TRUE; > + order_nest_info->nest_table_cols.push_back(item, thd->mem_root); > + } > + } > + > + /* > + TODO also add ORDER ITEMS that are expressions, only fields are covered above > + */ > + DBUG_ASSERT(!tab->table); > + > + order_nest_info->tmp_table_param.init(); > + order_nest_info->tmp_table_param.bit_fields_as_long= TRUE; > + order_nest_info->tmp_table_param.field_count= order_nest_info->nest_table_cols.elements; > + order_nest_info->tmp_table_param.force_not_null_cols= FALSE; > + > + const LEX_CSTRING order_nest_name= { STRING_WITH_LEN("order-nest") }; > + if (!(tab->table= create_tmp_table(thd, &order_nest_info->tmp_table_param, > + order_nest_info->nest_table_cols, (ORDER*) 0, > + FALSE /* distinct */, > + 0, /*save_sum_fields*/ > + thd->variables.option_bits | TMP_TABLE_ALL_COLUMNS, > + HA_POS_ERROR /*rows_limit */, > + &order_nest_name))) > + return TRUE; /* purecov: inspected */ > + > + //tab->tmp_table_param= &order_nest_info->tmp_table_param; > + tab->type= JT_ALL; > + > + /* > + add_sort_to_table(); > + */ > + > + return false; > +} > + > static int > return_zero_rows(JOIN *join, select_result *result, List<TABLE_LIST> &tables, > List<Item> &fields, bool send_row, ulonglong select_options, > @@ -28550,6 +28821,26 @@ select_handler *SELECT_LEX::find_select_ > return 0; > } > > +double postjoin_oper_cost(THD *thd, double join_record_count, uint rec_len, uint idx) The name of the function is confusing. I see the function is modeled after spl_postjoin_oper_cost(), but that name is not readable either. Is this the cost of using a sort nest? > +{ > + double cost= 0; > + /* > + For only one table in the order_nest, we don't need a fill the temp table, we can > + just read the data into the filesort buffer and read the sorted data from the buffers. > + */ > + if (idx) Comparing idx with zero seems odd here. If we check whether this is the first table, we should compare with join->const_tables (This applies to spl_postjoin_oper_cost, too. Need to ask Igor about that one). > + cost= get_tmp_table_write_cost(thd, join_record_count,rec_len) * > + join_record_count; // cost to fill tmp table > + > + cost+= get_tmp_table_lookup_cost(thd, join_record_count,rec_len) * > + join_record_count; // cost to perform post join operation used here > + cost+= get_tmp_table_lookup_cost(thd, join_record_count, rec_len) + > + (join_record_count == 0 ? 0 : > + join_record_count * log2 (join_record_count)) * > + SORT_INDEX_CMP_COST; // cost to perform sorting > + return cost; > +} > + > > /** > @} (end of group Query_Optimizer) > diff -urpN '--exclude=.*' 10.5-mdev8306-orig/sql/sql_select.h 10.5-mdev8306/sql/sql_select.h > --- 10.5-mdev8306-orig/sql/sql_select.h 2019-07-10 11:34:00.260257044 +0300 > +++ 10.5-mdev8306/sql/sql_select.h 2019-07-10 11:34:15.448604784 +0300 > @@ -289,13 +289,14 @@ typedef struct st_join_table { > > /* TRUE <=> This join_tab is inside an SJM bush and is the last leaf tab here */ > bool last_leaf_in_bush; > - > + > /* > ptr - this is a bush, and ptr points to description of child join_tab > range > NULL - this join tab has no bush children > */ > JOIN_TAB_RANGE *bush_children; > + JOIN_TAB_RANGE *order_nest_children; This is not yet used anywhere? Is there any reason to have both bush_children and order_nest_children? I would suggest using bush_children pointer for both SJ-Materialization nests and sort nests. Another variable in the JOIN_TAB will be used to specify what kind of nest it is. (One JOIN_TAB cannot be both a SJ-Mat nest and a sort nest). > > /* Special content for EXPLAIN 'Extra' column or NULL if none */ > enum explain_extra_tag info; > @@ -524,6 +525,12 @@ typedef struct st_join_table { > /* Becomes true just after the used range filter has been built / filled */ > bool is_rowid_filter_built; > > + /* > + Set to true if we consider creating a nest for a prefix of the JOIN order > + that satisfies the ordering > + */ > + bool is_order_nest; This is a member of JOIN_TAB. If we get here, we are not considering, we've firmly decided to use a sort nest. I would prefer the name "sort nest" rather than order_nest. > + > void build_range_rowid_filter_if_needed(); > > void cleanup(); > @@ -991,6 +998,9 @@ typedef struct st_position > /* Cost info for the range filter used at this position */ > Range_rowid_filter_cost_info *range_rowid_filter_info; > > + /* Flag to be set to TRUE if the join prefix satisfies the ORDER BY CLAUSE */ > + bool ordering_achieved; As far as I understand, this is set to true if we have decided to add a sort_nest. We can add a sort nest when the join prefix covers the ORDER BY, but we don't have to. So, the name is confusing and should be changed to something like "sort_nest_operation_here" > + > } POSITION; > > typedef Bounds_checked_array<Item_null_result*> Item_null_array; BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

1 0

[Maria-developers] INSERT ... RETURNING Initial code review
by Vicențiu Ciorbaru 10 Jul '19

10 Jul '19

Hi Rucha! Here is the promised review. I have run the main test suite locally and have noticedthat some tests fail if you run with a debug build. The failure is an assertionfailure on the protocol level. This means you forgot to call a my_ok or my_errorat an appropriate time. I suggest you try and find which one of your commitsintroduces this bug. A handy tool is `git bisect`. It's a great time to learnabout it, if you have not used it before. There are quite a few other things to fix, but first, let's tackle theinitial review comments. Now, on to the actual code. Your text editor inserts tabs instead of spaces. Please configure it to onlyinsert tabs and have the "tab" size be equal to 2 spaces. > diff --git a/mysql-test/main/insert_parser.test b/mysql- test/main/insert_parser.test> new file mode 100644> index 00000000000..5c1f9408a96> --- /dev/null> +++ b/mysql- test/main/insert_parser.test> @@ -0,0 +1,64 @@> +#> +# Syntax check for INSERT...RETURNING> +#> +> +--echo # RETURNING * and RETURNING <COLUMN_NAME> should work. Trailing whitespace here, please fix.> + --echo # Other cases with RETURNING should output syntax error.> +> + --disable_warnings> +drop table if exists t1,t2;> + --enable_warningsTest cases in MariaDB are written such that they do not leave behind anytables or other elements from running the test. Thus, such statements arenot required (and actively discouraged) in new test cases. If tables fromprevious tests are present and mysql-test-run (mtr) does not mark the test asfailed then inserting such statements will hide the mysql-test-run bug.Please delete.> +> +CREATE TABLE t1(id1 INT, val1 VARCHAR(1));> +--echo Table t1 created successfully. Fields: id1 INT, val1 VARCHAR(1);This comment is not necessary, please delete. The fact that mtr passed meansthe command was succesful. Also, when adding comments, I recommend you prefixthem with #, like you did for the start of the test.> +> +CREATE TABLE t2(id2 INT, val2 VARCHAR(1));> +--echo Table t2 created successfully. Fields: id2 INT, val2 VARCHAR(1);Same as previous comment, not necessary, please delete.> +> +#> +#Simple insert statement> +#> +--echo #Simple insert statementOnly keep one, I suggest the --echo variant, but put a space after #.I also like to add 2 extra --echo # like so:--echo #--echo # Simple insert statement--echo # This makes the result file easier to read.> +INSERT INTO t1 (id1, val1) VALUES (1, 'a');> +INSERT INTO t1 (id1, val1) VALUES (2, 'b') RETURNING *;> +INSERT INTO t1 (id1, val1) VALUES (3, 'c') RETURNING id1, val1;> +INSERT INTO t1 (id1, val1) VALUES (3, 'c') RETURNING id1 AS id;> +SELECT * FROM t1;> +TRUNCATE TABLE t1;> +SELECT * FROM t1;> +> +#> +#multiple values in one insert statement> +#> +--echo #multiple values in one insert statementSame as above.> +INSERT INTO t1 VALUES (1,'a'),(2,'b');> +INSERT INTO t1 VALUES (3,'c'),(4,'d') RETURNING *;> +INSERT INTO t1 VALUES (5,'e'),(6,'f') RETURNING id1, val1;> +INSERT INTO t1 VALUES (5,'e'),(6,'f') RETURNING id1 AS id;> +> +#> +#INSERT...ON DULPICATE KEY UPDATE> +#> +--echo # INSERT...ON DULPICATE KEY UPDATESame as above. Typo too.> +CREATE TABLE ins_duplicate (id INT PRIMARY KEY, val VARCHAR(1));> +INSERT INTO ins_duplicate VALUES (1,'a');> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='b' RETURNING id,val;> +INSERT INTO ins_duplicate VALUES (3,'b') ON DUPLICATE KEY UPDATE val='c' RETURNING *;> +INSERT INTO ins_duplicate VALUES (4,'b') ON DUPLICATE KEY UPDATE val='d' RETURNING id AS id1;> +#> +# INSERT...SET> +#> +> +--echo # INSERT...SETSame as above.> +INSERT INTO t1 SET id1 = 1, val1 = 'a'; > +INSERT INTO t1 SET id1 = 2, val1 = 'b' RETURNING *;> +INSERT INTO t1 SET id1 = 3, val1 = 'c' RETURNING id1,val1;> +INSERT INTO t1 SET id1 = 3, val1 = 'c' RETURNING id1 AS id;> +> +DROP TABLE t1; > +DROP TABLE t2;> +DROP TABLE ins_duplicate;> +> +#> +--echo #End of test case > +#This is superfluous, please delete everything after the last DROP TABLE.> \ No newline at end of file> diff --git a/mysql- test/main/insert_parser_1.test b/mysql-test/main/insert_parser_1.test> new file mode 100644> index 00000000000..f83c288a800> --- /dev/null> +++ b/mysql-test/main/insert_parser_1.test> @@ -0,0 +1,75 @@Hmm, this file does not have a corresponding .result file. It's almost a copyof the previous one and it has windows line endings. It's the only one thathas those. I suggest you move the statements not present in insert_parser.testthere and delete this file entirely.> +#> +# Syntax check for INSERT...RETURNING> +#> +> +--disable_warnings> +drop table if exists t1,t2;> +--enable_warnings> +> +CREATE TABLE t1(id1 INT, val1 VARCHAR(1));> +--echo Table t1 created successfully. Fields: id1 INT, val1 VARCHAR(1);> +> +CREATE TABLE t2(id2 INT, val2 VARCHAR(1));> + --echo Table t2 created successfully. Fields: id2 INT, val2 VARCHAR(1);> +> +#> +#Simple insert statement> +#> +--echo #Simple insert statement> +INSERT INTO t1 (id1, val1) VALUES (1, 'a');> +INSERT INTO t1 (id1, val1) VALUES (2, 'b') RETURNING *;> +INSERT INTO t1 (id1, val1) VALUES (3, 'c') RETURNING id1, val1;> +INSERT INTO t1 (id1, val1) VALUES (4, 'd') RETURNING id1 as id;> +INSERT INTO t1 (id1, val1) VALUES (5, 'e') RETURNING id1 && id1;> +INSERT INTO t1 (id1, val1) VALUES (6, 'f') RETURNING id1 + id1;> +INSERT INTO t1 (id1, val1) VALUES (7, 'g') RETURNING id1 | id1;> +SELECT * FROM t1;> +TRUNCATE TABLE t1;> +SELECT * FROM t1;> +> +#> +#multiple values in one insert statement> +#> +--echo #multiple values in one insert statement> +INSERT INTO t1 VALUES (1,'a'),(2,'b');> +INSERT INTO t1 VALUES (3,'c'),(4,'d') RETURNING *;> +INSERT INTO t1 VALUES (5,'e'),(6,'f') RETURNING id1, val2;> +INSERT INTO t1 VALUES (7,'g'),(8,'h') RETURNING id1 AS id;> +INSERT INTO t1 VALUES (9,'i'),(10,'j') RETURNING id1 | id1 ;> +INSERT INTO t1 VALUES (11,'k'),(12,'l') RETURNING id1 + id1;> +INSERT INTO t1 VALUES (13,'m'),(14,'n') RETURNING id1 && id1;> +SELECT * FROM t1;> +> +#> +#INSERT...ON DULPICATE KEY UPDATE> +#> +--echo # INSERT...ON DULPICATE KEY UPDATE> +CREATE TABLE ins_duplicate (id INT PRIMARY KEY, val VARCHAR(1));> +INSERT INTO ins_duplicate VALUES (1,'a');> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='b' RETURNING id,val;> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='c' RETURNING *;> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='c' RETURNING id AS id1;> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='c' RETURNING id | id;> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='c' RETURNING id && id;> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='c' RETURNING id +id;> +SELECT * FROM ins_duplicate;> +> +#> +# INSERT...SET> +#> +--echo # INSERT...SET> +INSERT INTO t1 SET id1 = 1, val1 = 'a'; > +INSERT INTO t1 SET id1 = 2, val1 = 'b' RETURNING *;> +INSERT INTO t1 SET id1 = 3, val1 = 'c' RETURNING id1,val1;> +INSERT INTO t1 SET id1 = 3, val1 = 'c' RETURNING id1 as id;> +INSERT INTO t1 SET id1 = 3, val1 = 'c' RETURNING id1 | id1;> +INSERT INTO t1 SET id1 = 3, val1 = 'c' RETURNING id1 && id1;> +INSERT INTO t1 SET id1 = 3, val1 = 'c' RETURNING id1 + id1;> +> +DROP TABLE t1; > +DROP TABLE t2;> +DROP TABLE ins_duplicate;> +> +#> +--echo #End of test case > +#> \ No newline at end of file> diff --git a/mysql- test/main/insert_parser_sel.test b/mysql- test/main/insert_parser_sel.test> new file mode 100644> index 00000000000..f144cfc48ec> --- /dev/null> +++ b/mysql- test/main/insert_parser_sel.test> @@ -0,0 +1,45 @@> +#> +# Syntax check for INSERT...RETURNING> +#> +> +--disable_warnings> +drop table if exists t1,t2;> +--enable_warningsSimilar to insert_parser.test, these lines are not necessary.> +> +CREATE TABLE t1(id1 INT, val1 VARCHAR(1));> +--echo Table t1 created successfully. Fields: id1 INT, val1 VARCHAR(1);See insert_parser.test comments.> +> +CREATE TABLE t2(id2 INT, val2 VARCHAR(1));> +--echo Table t2 created successfully. Fields: id2 INT, val2 VARCHAR(1);See insert_parser.test comments.> +> +#> +#Simple insert statement> +#> +--echo #Simple insert statementSee insert_parser.test comments.> +INSERT INTO t1 (id1, val1) VALUES (1, 'a');> +INSERT INTO t1 (id1, val1) VALUES (2, 'b');> +INSERT INTO t1 (id1, val1) VALUES (3, 'c');> +INSERT INTO t1 (id1, val1) VALUES (4, 'd');> +INSERT INTO t1 (id1, val1) VALUES (5, 'e');> +INSERT INTO t1 (id1, val1) VALUES (6, 'f');> +INSERT INTO t1 (id1, val1) VALUES (7, 'g');> +SELECT * FROM t1;> +TRUNCATE TABLE t1;> +SELECT * FROM t1;> +> +#> +# INSERT...SELECT> +#> +--echo #INSERT...SELECTSee insert_parser.test comments.> +INSERT INTO t2(id2,val2) SELECT * FROM t1;> +INSERT INTO t2(id2,val2) SELECT * FROM t1 WHERE id1=1 RETURNING *;> +INSERT INTO t2(id2,val2) SELECT * FROM t1 WHERE id1=2 RETURNING id2,val2;> +INSERT INTO t2(id2,val2) SELECT * FROM t1 WHERE id1=3 RETURNING id2 AS id;> +INSERT INTO t2(id2,val2) SELECT * FROM t1 WHERE id1 IN (SELECT id1 FROM t1 WHERE id1 IN (3,4,5) );Ok, this last line does not test RETURNING at all, what was the intent?Also, please wrap queries to maximum 80 lines. Find the right way to indentthe SQL query on multiple lines to make it easier to read.> +SELECT * FROM t2;> +DROP TABLE t1; > +DROP TABLE t2;> +> +#> +--echo #End of test case > +#See insert_parser.test comments.> \ No newline at end of file> diff --git a/mysql-test/main/insert_returning_complex.test b/mysql- test/main/insert_returning_complex.test> new file mode 100644> index 00000000000..59ef3342e76> --- /dev/null> +++ b/mysql- test/main/insert_returning_complex.test> @@ -0,0 +1,117 @@> +#> +# Test for INSERT...RETURNING with virual cols,operators and # subqueries> +#> +> +--disable_warnings> +drop table if exists t1,t2,ins_duplicate;> + --enable_warningsSee insert_returning.testI would merge this test into the insert_returning.test file. No need to split them up.> +> +CREATE TABLE t1(id1t1 INT, id2t1 INT, valt1 VARCHAR(1));> +CREATE TABLE t2(id1t2 INT, id2t2 INT, valt2 VARCHAR(1));> +INSERT INTO t2 VALUES (1,2,'a'),(2,3,'b'),(3,4,'c'),(4,5,'d'),(5,6,'e'),(6,7,'f'),(7,8,'g'),( 8,9,'h'),(9,10,'i'),(10,11,'j');Please wrap this line to 80 characters max.> +> +--echo Table t1 created successfully. Fields: id1 INT, id2 INT, val VARCHAR(1);Superfluous.> +> +#> +#Simple insert statement> +#> +--echo #Simple insert statementSee insert_returning.test comments.> +INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (1, 2, 'a');> +INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (2, 3, 'b') RETURNING *;> +INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (3, 4, 'c') RETURNING id1t1, valt1;> +INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (4, 5, 'd') RETURNING id1t1 AS id;> +INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (5, 6, 'e') RETURNING id1t1+id2t1 AS total;> +INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (6, 7, 'f') RETURNING id1t1 & id2t1;> +INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (7, 8, 'g') RETURNING id1t1 || id2t1;> +INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (8,9,'h') RETURNING id1t1,UPPER(valt1);> +INSERT INTO t1(id1t1,id2t1,valt1) VALUES (9,10,'i') RETURNING (SELECT GROUP_CONCAT(valt2) FROM t2 WHERE id1t2=1);> +INSERT INTO t1(id1t1,id2t1,valt1) VALUES(10,11,'j') RETURNING (SELECT GROUP_CONCAT(valt2) FROM t2 GROUP BY id1t2 HAVING id1t2=id1t2+1);Ok, please wrap all such lines to 80 characters to make them easier to parse.Second, can we test with subqueries that return multiple rows, for example:SELECT GROUP_CONCAT(valt2) FROM t2 group by id1t2.We should return an error here, but this should be covered in the test casetoo.> +INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (8, 9, 'h') RETURNING id1t1,(SELECT id2t2 FROM t2 WHERE valt2='b');Can you also test a subquery that uses columns from t1?Example:INSERT INTO t1 (id1t1, id2t1, valt1) VALUES (8, 9, 'h') RETURNING id1t1, (SELECT id2t2 + id1t1 FROM t2 WHERE valt2='b');> > +SELECT * FROM t1;> +TRUNCATE TABLE t1;> +SELECT * FROM t1;> +> +> +#> +#multiple values in one insert statement> +#> +--echo #multiple values in one insert statementSee insert_returning.test comments.> +INSERT INTO t1 VALUES (1, 2, 'a'),(2, 3, 'b');> +INSERT INTO t1 VALUES (3, 4, 'c'),(4, 5, 'd') RETURNING *;> +INSERT INTO t1 VALUES (5, 6, 'e'),(6, 7, 'f') RETURNING id2t1, valt1;> +INSERT INTO t1 VALUES (7, 8, 'g'),(8, 9, 'h') RETURNING id2t1 AS id;> +INSERT INTO t1 VALUES (9, 10, 'i'),(10, 11, 'j') RETURNING id2t1+id1t1 as total;> +INSERT INTO t1 VALUES (11, 12, 'k'),(12, 13, 'l') RETURNING id2t1 & id1t1;> +INSERT INTO t1 VALUES (5, 6, 'e'),(2, 3, 'f') RETURNING id2t1 || id1t1;> +INSERT INTO t1 VALUES (13,14,'m'),(14,15,'n') RETURNING id1t1,UPPER(valt1);> +INSERT INTO t1 VALUES (15,16,'o'),(16,17,'p') RETURNING (SELECT GROUP_CONCAT(valt2) FROM t2 WHERE id1t2=1);> +INSERT INTO t1 VALUES (17,18,'q'),(18,19,'r') RETURNING (SELECT GROUP_CONCAT(valt2) FROM t2 GROUP BY id1t2 HAVING id1t2=id1t2+1)> +INSERT INTO t1 VALUES (19,20,'r'),(20,21,'s') RETURNING id1t1,(SELECT id2t2 FROM t2 WHERE valt2='b');Same comments as for single-value insert.> +SELECT * FROM t1;> +TRUNCATE TABLE t1;> +> +#> +#INSERT...ON DULPICATE KEY UPDATE> +#> +--echo # INSERT...ON DULPICATE KEY UPDATETypo DULPICATE -> DUPLICATE. Also, see comments from insert_returning.test.> +CREATE TABLE ins_duplicate (id1 INT PRIMARY KEY, id2 INT, val VARCHAR(1));> +INSERT INTO ins_duplicate VALUES (1,2,'a');> +INSERT INTO ins_duplicate VALUES (2,3,'b') ON DUPLICATE KEY UPDATE val='b' RETURNING id1,val;> +INSERT INTO ins_duplicate VALUES (2,4,'b') ON DUPLICATE KEY UPDATE val='c' RETURNING *;> +INSERT INTO ins_duplicate VALUES (2,5,'b') ON DUPLICATE KEY UPDATE val='d' RETURNING id1+id2 AS total;> +INSERT INTO ins_duplicate VALUES (2,6,'b') ON DUPLICATE KEY UPDATE val='e' RETURNING id1 || id2;> +INSERT INTO ins_duplicate VALUES (2,7,'b') ON DUPLICATE KEY UPDATE val='f' RETURNING id1 & id2;> +INSERT INTO ins_duplicate VALUES (2,8,'b') ON DUPLICATE KEY UPDATE val='g' RETURNING id1 AS id;> +INSERT INTO ins_duplicate VALUES (2,9,'b') ON DUPLICATE KEY UPDATE val='h' RETURNING id1,UPPER(val);> +INSERT INTO ins_duplicate VALUES (2,10,'b') ON DUPLICATE KEY UPDATE val='i' RETURNING (SELECT GROUP_CONCAT(valt2) FROM t2 WHERE id1t2=1);> +INSERT INTO ins_duplicate VALUES (2,11,'b') ON DUPLICATE KEY UPDATE val='j' RETURNING (SELECT GROUP_CONCAT(valt2) FROM t2 GROUP BY id1t2 HAVING id1t2=id1t2+1);> +INSERT INTO ins_duplicate VALUES (2,12,'b') ON DUPLICATE KEY UPDATE val='k' RETURNING id1,(SELECT id2t2 FROM t2 WHERE valt2='b');> +SELECT * FROM ins_duplicate;> +> +#> +# INSERT...SET> +#> +> +--echo # INSERT...SET> +INSERT INTO t1 SET id1t1=1, id2t1=2, valt1='a'; > +INSERT INTO t1 SET id1t1=2, id2t1=3, valt1='b' RETURNING *;> +INSERT INTO t1 SET id1t1=3, id2t1=4, valt1='c' RETURNING valt1;> +INSERT INTO t1 SET id1t1=4, id2t1=5, valt1='d' RETURNING id1t1+id2t1 AS total;> +INSERT INTO t1 SET id1t1=5, id2t1=6, valt1='e' RETURNING id1t1 & id2t1;> +INSERT INTO t1 SET id1t1=6, id2t1=7, valt1='f' RETURNING id1t1 || id2t1;> +INSERT INTO t1 SET id1t1=7, id2t1=8, valt1='g' RETURNING valt1 AS letter;> +INSERT INTO t1 SET id1t1=8, id2t1=9, valt1='h' RETURNING id1t1,UPPER(valt1);> +INSERT INTO t1 SET id1t1=9, id2t1=10, valt1='i' RETURNING (SELECT GROUP_CONCAT(valt2) FROM t2 WHERE id1t2=1);> +INSERT INTO t1 SET id1t1=10, id2t1=11, valt1='j' RETURNING (SELECT GROUP_CONCAT(valt2) FROM t2 GROUP BY id1t2 HAVING id1t2=id1t2+1);> +INSERT INTO t1 SET id1t1=11, id2t1=12, valt1='k' RETURNING id1t1,(SELECT id2t2 FROM t2 WHERE valt2='b'); > +SELECT * FROM t1;> +TRUNCATE TABLE t1;> +SELECT * FROM t1;> +> +#> +# INSERT...SELECT> +#> +--echo # INSERT...SELECT> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=1;> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=2 RETURNING *;> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=3 RETURNING valt1;> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=4 RETURNING id1t1+id2t1 AS total;> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=5 RETURNING id1t1 & id2t1;> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=6 RETURNING valt1 AS letter;> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=2 RETURNING id1t1 || id2t1;> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=7 RETURNING id1t1,UPPER(valt1);> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=8> +RETURNING (SELECT GROUP_CONCAT(val) FROM ins_duplicate WHERE id1=1);> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=9> +RETURNING (SELECT GROUP_CONCAT(val) FROM ins_duplicate GROUP BY id1 HAVING id1=id1+1);> +INSERT INTO t1(id1t1, id2t1, valt1) SELECT * FROM t2 WHERE id1t2=10 RETURNING id1t1,(SELECT id2 FROM ins_duplicate WHERE valt2='b');> +SELECT* FROM t1;> +> +--echo Droping t1 and ins_duplicate> +DROP TABLE t1;> +DROP TABLE t2; > +DROP TABLE ins_duplicate;> +> +#> +--echo #End of test case > +# > \ No newline at end of file> Binary files /dev/null and b/mysql- test/main/insert_returning_datatypes.result differOk, why is this a binary file for git? Something went wrong here.> diff --git a/mysql- test/main/insert_returning_datatypes.test b/mysql- test/main/insert_returning_datatypes.test> new file mode 100644> index 00000000000..803c8326b6f> --- /dev/null> +++ b/mysql- test/main/insert_returning_datatypes.test> @@ -0,0 +1,92 @@> +#> +# Tests for DELETE FROM <table> ... RETURNING <expr>,...> +# What is DELETE FROM .. RETURNING doing here? :)> +> +--disable_warnings> +drop table if exists t1,t2;> +--enable_warningsI vote for merging this into the previous tests.> +> +CREATE TABLE t1(num_int1 INT(2) PRIMARY KEY, > + num_bit1 BIT(2),> + num_float1 FLOAT(5,2),> + num_double1 DOUBLE(5,2),> + char_enum1 ENUM('A','B','C','D'),> + char_set1 SET('a','b','c','d','e'),> + str_varchar1 VARCHAR(2),> + str_binary1 BINARY(2),> + d1 DATE,> + dt1 DATETIME,> + ts1 TIMESTAMP,> + y1 YEAR,> + b1 BOOL);> +> +CREATE TABLE t2(num_int2 INT(2) PRIMARY KEY, > + num_bit2 BIT(2),> + num_float2 FLOAT(5,2),> + num_double2 DOUBLE(5,2),> + char_enum2 ENUM('A','B','C','D'),> + char_set2 SET('a','b','c','d','e'),> + str_varchar2 VARCHAR(2),> + str_binary2 BINARY(2),> + d2 DATE,> + dt2 DATETIME,> + ts2 TIMESTAMP,> + y2 YEAR,> + b2 BOOL);> +> +> +#> +# SIMLPE INSERT STATEMENT> +#Typo, but please delete this anayway.> +--echo # SIMPLE INSERT STATEMENTSee other test comments.> +INSERT INTO t1(num_int1,num_bit1,num_float1,num_double1,char_enum1,char_set1,str_va rchar1, str_binary1,d1,dt1,ts1,y1,b1) VALUES(1,0, 123.45, 123.55, 'A','b,e', 'V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,0) RETURNING *;> +> +INSERT INTO t1(num_int1,num_bit1,num_float1,num_double1,char_enum1,char_set1,str_va rchar1, str_binary1,d1,dt1,ts1,y1,b1) VALUES(2,0, 123.45, 123.55, 'A','b,e', 'V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,0) RETURNING num_int1,num_bit1,d1,dt1;> +> +> +> +#> +# MULTIPLE ROWS IN SINGLE STATEMENT > +#> +--echo #Multiple rows inserted in one statement> +INSERT INTO t1 VALUES(3,0, 123.45, 123.55, 'A','b,e', 'V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,0),(4,0, 123.45, 123.55, 'A','b,e', 'V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,1),(5,0, 123.45, 123.55, 'A','b,e','V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,1) RETURNING *;> +> +INSERT INTO t1 VALUES(6,0, 123.45, 123.55, 'A','b,e', 'V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,0),(7,0, 123.45, 123.55, 'A','b,e', 'V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,1),(8,0, 123.45, 123.55, 'A','b,e','V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,1) RETURNING char_set1,ts1,y1;> +> +> +> +#> +#INSERT...SET...RETURNING> +#> +--echo #INSERT...SET...RETURNING> +INSERT INTO t1 SET num_int1=9,num_bit1=0,num_float1=124.67,num_double1=231.12,char_enum1=' B',char_set1='a,d,e',str_varchar1='AB',str_binary1='01', d1='120314',dt1="2012-04-19 13:08:22",ts1='2001-07-22 12:12:1',y1=2014,b1=1 RETURNING *;> +> +INSERT INTO t1 SET num_int1=10,num_bit1=0,num_float1=124.67,num_double1=231.12,char_enum1= 'B',char_set1='a,d,e',str_varchar1='AB',str_binary1='01',d1='120314',dt 1="2012-04-19 13:08:22",ts1='2001-07-22 12:12:1',y1=2014,b1=1 RETURNING b1,char_enum1;> +> +> +> +#> +# INSERT...ON DUPLICATE KEY UPDATE> +# > +--echo # INSERT...ON DUPLCATE KEY UPDATE...RETURNING> +INSERT INTO t1(num_int1,num_bit1,num_float1,num_double1,char_enum1,char_set1,str_va rchar1, str_binary1,d1,dt1,ts1,y1,b1) VALUES (10,0, 123.45, 123.55, 'C','b,e', 'V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,0) ON DUPLICATE KEY UPDATE num_float1=111.111 RETURNING *;> +> +INSERT INTO t1(num_int1,num_bit1,num_float1,num_double1,char_enum1,char_set1,str_va rchar1, str_binary1,d1,dt1,ts1,y1,b1) VALUES (11,0, 123.45, 123.55, 'A','b,e', 'V','10','120314',"2012-04-19 13:08:22", '2001-07-22 12:12:12',2012,0) ON DUPLICATE KEY UPDATE char_set1='a,b' RETURNING char_set1,num_int1;> +> +> +> +#> +# INSERT...SELECT...RETURNING> +#> +--echo # INSERT...SELECT...RETURNING> +INSERT INTO t2 (num_int2,num_bit2,num_float2,num_double2,char_enum2,char_set2,str_varc har2, str_binary2,d2,dt2,ts2,y2,b2) SELECT * FROM t1 RETURNING *;> +TRUNCATE TABLE t2;> +INSERT INTO t2 (num_int2,num_bit2,num_float2,num_double2,char_enum2,char_set2,str_varc har2, str_binary2,d2,dt2,ts2,y2,b2) SELECT * FROM t1 RETURNING num_double2,str_varchar2,str_binary2;> +> +> +DROP TABLE t1;> +DROP TABLE t2;> +> +#> +#END OF TEST CASE> +# > \ No newline at end of file> diff --git a/mysql-test/main/insert_returning_err.test b/mysql- test/main/insert_returning_err.test> new file mode 100644> index 00000000000..4af798e5c68> --- /dev/null> +++ b/mysql- test/main/insert_returning_err.test> @@ -0,0 +1,97 @@> +#> +# Test for checking error message for INSERT...RETURNING> +#> +> +#INSERT INTO <table> ... RETURNING <not existing col>> +#INSERT INTO <table> ... RETURNING <expr with aggr function>> +#INSERT INTO ... RETURNING subquery with more than 1 row> +#INSERT INTO ... RETURNING operand should contain 1 colunm(s)> +#INSERT INTO ... RETURNING operand should contain 1 colunm(s)> +> +--disable_warnings> +drop table if exists t1,t2;> +--enable_warnings> +> +CREATE TABLE t1(id1 INT,val1 VARCHAR(1));> +CREATE TABLE t2(id2 INT,val2 VARCHAR(1));> +CREATE TABLE ins_duplicate (id INT PRIMARY KEY, val VARCHAR(1));> +> +INSERT INTO t1 VALUES(1,'a'),(2,'b'),(3,'c');> +> +#> +# SIMLPE INSERT STATEMENT> +#> +--error ER_BAD_FIELD_ERROR> +INSERT INTO t2(id2,val2) VALUES(1,'a') RETURNING id1;> +--error ER_INVALID_GROUP_FUNC_USE> +INSERT INTO t2(id2,val2) values(2,'b') RETURNING MAX(id2);> +--error ER_SUBQUERY_NO_1_ROW> +INSERT INTO t2(id2,val2) VALUES(3,'c') RETURNING (SELECT id1 FROM t1);Ok, so you do test this, good. What about a subquery using a column from t2?> +--error ER_OPERAND_COLUMNS> +INSERT INTO t2(id2,val2) VALUES(4,'d') RETURNING (SELECT * FROM t1);This query fails specifically because there are multiple columns in the subselect.What happens if there is only one column?> +--error ER_OPERAND_COLUMNS> +INSERT INTO t2(id2,val2) VALUES(4,'d') RETURNING (SELECT * FROM t2);> +> +#> +# Multiple rows in single insert statement> +#> +--error ER_BAD_FIELD_ERROR> +INSERT INTO t2 VALUES(1,'a'),(2,'b') RETURNING id1;> +--error ER_INVALID_GROUP_FUNC_USE> +INSERT INTO t2 VALUES(3,'c'),(4,'d') RETURNING MAX(id2);> +--error ER_SUBQUERY_NO_1_ROW> +INSERT INTO t2 VALUES(5,'c'),(6,'f') RETURNING (SELECT id1 FROM t1);> +--error ER_OPERAND_COLUMNS> +INSERT INTO t2 VALUES(7,'g'),(8,'h') RETURNING (SELECT * FROM t1);> +--error ER_OPERAND_COLUMNS> +INSERT INTO t2 VALUES(7,'g'),(8,'h') RETURNING (SELECT * FROM t2);> +> +#> +# INSERT ... SET> +#> +--error ER_BAD_FIELD_ERROR> +INSERT INTO t2 SET id2=1, val2='a' RETURNING id1;> +--error ER_INVALID_GROUP_FUNC_USE> +INSERT INTO t2 SET id2=2, val2='b' RETURNING MAX(id2);> +--error ER_SUBQUERY_NO_1_ROW> +INSERT INTO t2 SET id2=3, val2='c' RETURNING (SELECT id1 FROM t1);> +--error ER_OPERAND_COLUMNS> +INSERT INTO t2 SET id2=4, val2='d' RETURNING (SELECT * FROM t1);> +--error ER_OPERAND_COLUMNS> +INSERT INTO t2 SET id2=4, val2='d' RETURNING (SELECT * FROM t2);> +> +#> +#INSERT...ON DUPLICATE KEY UPDATE> +#> + --error ER_BAD_FIELD_ERROR> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='b' RETURNING id1;> +--error ER_INVALID_GROUP_FUNC_USE> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='b' RETURNING MAX(id);> +--error ER_SUBQUERY_NO_1_ROW> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='b' RETURNING (SELECT id1 FROM t1);> +--error ER_OPERAND_COLUMNS> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='b' RETURNING (SELECT * FROM t1);> +--error ER_OPERAND_COLUMNS> +INSERT INTO ins_duplicate VALUES (2,'b') ON DUPLICATE KEY UPDATE val='b' RETURNING (SELECT * FROM ins_duplicate);> +> +#> +#INSERT...SELECT...RETURNING> +#Use --echo # for the lines above please.> +--error ER_BAD_FIELD_ERROR> +INSERT INTO t2(id2, val2) SELECT * FROM t1 WHERE id1=1 RETURNING id1;> +--error ER_INVALID_GROUP_FUNC_USE> +INSERT INTO t2(id2, val2) SELECT * FROM t1 WHERE id1=2 RETURNING MAX(id2);> +--error ER_SUBQUERY_NO_1_ROW> +INSERT INTO t2(id2, val2) SELECT * FROM t1 WHERE id1=2 RETURNING (SELECT id1 FROM t1);> +--error ER_OPERAND_COLUMNS> +INSERT INTO t2(id2, val2) SELECT * FROM t1 WHERE id1=2 RETURNING (SELECT * FROM t1);> +--error ER_OPERAND_COLUMNS> +INSERT INTO t2(id2, val2) SELECT * FROM t1 WHERE id1=2 RETURNING (SELECT * FROM t2);> +> +DROP TABLE t1;> +DROP TABLE t2;> +DROP TABLE ins_duplicate;> +> +#> +# End of test case> +# > \ No newline at end of file> diff --git a/mysql- test/main/insert_returning_stmt.result b/mysql- test/main/insert_returning_stmt.result> new file mode 100644> index 00000000000..3b48cb11b28> --- /dev/null> +++ b/mysql- test/main/insert_returning_stmt.resultHow is this test different than previous ones? I am getting a bit confusedby the large number of test files that seem simillar.> \ No newline at end of file> diff --git a/mysql-test/main/insert_sel_returning.test b/mysql- test/main/insert_sel_returning.test> new file mode 100644> index 00000000000..8b4880e2064> --- /dev/null> +++ b/mysql- test/main/insert_sel_returning.test> diff --git a/sql/sql_class.h b/sql/sql_class.h> index 18ccf992d1e..8e67818f891 100644> --- a/sql/sql_class.h> +++ b/sql/sql_class.h> @@ -5475,13 +5475,15 @@ class select_dump :public select_to_file {> > class select_insert :public select_result_interceptor {> public:> + select_result* sel_result;> + bool with_returning_list;> TABLE_LIST *table_list;> TABLE *table;> List<Item> *fields;> ulonglong autoinc_value_of_last_inserted_row; // autogenerated or not> COPY_INFO info;> bool insert_into_view;> - select_insert(THD *thd_arg, TABLE_LIST *table_list_par,> + select_insert(bool with_ret_list, select_result* sel_ret_list, THD *thd_arg, TABLE_LIST *table_list_par,When adding parameters, please add them to the end of the function, not thestart. Do we need both with_returning_list and sel_result pointer? Can wenot use just the sel_result pointer and if it is null, we assume we do nothave a returning list?> TABLE *table_par, List<Item> *fields_par,> List<Item> *update_fields, List<Item> *update_values,> enum_duplicates duplic, bool ignore);> diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc> index f3a548d7265..6eb9883fe87 100644> --- a/sql/sql_insert.cc> +++ b/sql/sql_insert.cc> @@ -1236,26 +1261,38 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list,> if ((iteration * values_list.elements) == 1 && (!(thd->variables.option_bits & OPTION_WARNINGS) ||> !thd->cuted_fields))> {> - my_ok(thd, info.copied + info.deleted +> - ((thd- >client_capabilities & CLIENT_FOUND_ROWS) ?> - info.touched : info.updated),> - id);> + if (with_returning_list)> + result->send_eof();> + else > + {> + my_ok(thd, info.copied + info.deleted +> + ((thd->client_capabilities & CLIENT_FOUND_ROWS) ?> + info.touched : info.updated),> + id);> + }> }> e lse> {> char buff[160];> ha_rows updated=((thd- >client_capabilities & CLIENT_FOUND_ROWS) ?> info.touched : info.updated);> - if (ignore)> - sprintf(buff, ER_THD(thd, ER_INSERT_INFO), (ulong) info.records,> - (lock_type == TL_WRITE_DELAYED) ? (ulong) 0 :> - (ulong) (info.records - info.copied),> - (long) thd->get_stmt_da()- >current_statement_warn_count());> - else> - sprintf(buff, ER_THD(thd, ER_INSERT_INFO), (ulong) info.records,> - (ulong) (info.deleted + updated),> - (long) thd->get_stmt_da()- >current_statement_warn_count());> - ::my_ok(thd, info.copied + info.deleted + updated, id, buff);> + if (ignore)> + sprintf (buff, ER_THD(thd, ER_INSERT_INFO), (ulong)info.records,> + (lock_type == TL_WRITE_DELAYED) ? (ulong)0 :> + (ulong) (info.records - info.copied),> + (long)thd- >get_stmt_da()->current_statement_warn_count());> + else > + {> + if (with_returning_list)> + result- >send_eof();> + else > + {> + sprintf(buff, ER_THD(thd, ER_INSERT_INFO), (ulong)info.records,> + (ulong)(info.deleted + updated),> + (long)thd->get_stmt_da()- >current_statement_warn_count());> + ::my_ok(thd, info.copied + info.deleted + updated, id, buff);> + }> + }Uhm, careful here, you have changed the logic of when my_ok gets called! Please re-read this bit of code and try to understand where things havechanged. Also, looking at this, can you test INSERT IGNORE ... RETURNING too? I have a feeling it won't work with the current implementation.> }> thd->abort_on_warning= 0;> if (thd->lex- >current_select->first_cond_optimization)> @@ -1557,7 +1595,10 @@ bool mysql_prepare_insert(THD *thd, TABLE_LIST *table_list,> table_list->next_local= 0;> context- >resolve_in_table_list_only(table_list);> > - res= (setup_fields(thd, Ref_ptr_array(),> + res= ((with_returning_list?((sel_lex->with_wild && setup_wild(thd, table_list, sel_lex->returning_list, NULL, sel_lex->with_wild,> + &select_lex->hidden_bit_fields)) ||> + setup_fields(th d, Ref_ptr_array(),> + sel_lex->returning_list, MARK_COLUMNS_READ, 0, NULL, 0)):false)||setup_fields(thd, Ref_ptr_array(),Uhm, please fix the formatting for this code. The logic is also quite convoluted,can you turn this into something actually readable? Also, you are calling the same setup_wild && setup_fields call both withinmysql_prepare_insert, but also within select_result->prepare. Why 2 times?> *values, MARK_COLUMNS_READ, 0, NULL, 0) ||> check_insert_fields(thd, context->table_list, fields, *values,> !insert_into_view, 0, &map));> @@ -3557,12 +3598,13 @@ bool Delayed_insert::handle_inserts(void)> TRUE Error> */> > -bool mysql_insert_select_prepare(THD *thd)> +bool mysql_insert_select_prepare(THD *thd,select_result *sel_res)> {> LEX *lex= thd->lex;> SELECT_LEX *select_lex= lex- >first_select_lex();> DBUG_ENTER("mysql_insert_select_prepare");> -> + bool with_ret = lex->current_select- >returning_list.is_empty()?false:true;> + List<Item>& returning_list = thd->lex->current_select->returning_list;Not really a fan of this method. By doing this, it's going to be impossibleto extend INSERT .. RETURNING to work within subqueries.> > /*> SELECT_LEX do not belong to INSERT statement, so we can't add WHERE> @@ -3572,14 +3614,18 @@ bool mysql_insert_select_prepare(THD *thd)> if (mysql_prepare_insert(thd, lex- >query_tables,> lex->query_tables->table, lex->field_list, 0,> lex->update_list, lex- >value_list, lex->duplicates,> - &select_lex- >where, TRUE))> + &select_lex->where, TRUE,with_ret?select_lex:NULL))I don't like that we end up passing select_lex to mysql_prepare_insert.Can we make the arguments of this function be strictly what is necessary,not the full SELECT_LEX structure?> DBUG_RETURN(TRUE);> > + if (with_ret)> + (void )sel_res->prepare(returning_list, NULL);> +> DBUG_ASSERT(select_lex- >leaf_tables.elements != 0);> List_iterator<TABLE_LIST> ti(select_lex->leaf_tables);> TABLE_LIST *table;> uint insert_tables;> > +> if (select_lex- >first_cond_optimization)> {> /* Back up leaf_tables list. */> @@ -3630,6 +3677,8 @@ select_insert::select_insert(THD *thd_arg, TABLE_LIST *table_list_par,> info.update_values= update_values;> info.view= (table_list_par->view ? table_list_par : 0);> info.table_list= table_list_par;> + sel_result= result;> + with_returning_list= with_ret_list;Can we get rid of with_returning_list and just use sel_result? If it's null,with_returning_list should also be null.> }> > > @@ -3651,7 +3701,28 @@ select_insert::prepare(List<Item> &values, SELECT_LEX_UNIT *u)> */> lex->current_select= lex->first_select_lex();> > - res= (setup_fields(thd, Ref_ptr_array(),> + /*> + We want the returning_list to point to insert table. But the context is masked. > + So we swap it with the context saved during parsing stage. > + */> + if(with_returning_list) > + { swap_context(select_lex- >table_list.saved_first,select_lex->table_list.first);> + swap_context(select_lex->context.saved_table_list,select_lex- >context.table_list);> + swap_context(select_lex- >context.saved_name_resolution_table,select_lex- >context.first_name_resolution_table);> +> + res=((select_lex- >with_wild && setup_wild(thd, table_list, select_lex->returning_list, NULL, select_lex->with_wild,> + &select_lex- >hidden_bit_fields)) ||> + setup_fields(thd, Ref_ptr_array(),> + select_lex->returning_list, MARK_COLUMNS_READ, 0, NULL, 0));> +> + /*Swap it back to retore the previous state for the rest of the function*/> +> + swap_context(select_lex->table_list.saved_first,select_lex- >table_list.first);> + swap_context(select_lex- >context.saved_table_list,select_lex->context.table_list);> + swap_context(select_lex->context.saved_name_resolution_table, select_lex->context.first_name_resolution_table);> + }I am not sure this will work if you have columns with the same names, bothin t1 and t2 (your tests don't cover this).Ex:CREATE table t1 (pk int);CREATE table t2 (pk int);INSERT INTO t2 (pk) values (1), (2); INSERT INTO t1 SELECT * FROM t2 RETURNING pk; Also, what about INSERT INTO t1 SELECT .. FROM t1? Please add a test casefor this and see if the code still continues to work. Also, the saved_first introduction into SQL_I_List is really a hack.One needs to save the correct list of tables separately, properly. Pleasefind a solution without changing SQL_I_List. If this part gets you stuck,we should discuss it, but I want you first to try and find a proper solutionfirst.> +> + res= res || (setup_fields(thd, Ref_ptr_array(),> values, MARK_COLUMNS_READ, 0, NULL, 0) ||> check_insert_fields(thd, table_list, *fields, values,> !insert_into_view, 1, &map));> @@ -4046,7 +4136,10 @@ bool select_insert::send_ok_packet() {> thd- >first_successful_insert_id_in_prev_stmt :> (info.copied ? autoinc_value_of_last_inserted_row : 0));> > - ::my_ok(thd, row_count, id, message);> + if (with_returning_list)> + sel_r esult->send_eof();> + else> + ::my_ok(thd, row_count, id, message);I believe you need to return my_ok regardless.> > DBUG_RETURN(false);> }> diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc> index 01d0ed1c383..6cee252096b 100644> --- a/sql/sql_parse.cc> +++ b/sql/sql_parse.cc> @@ -4604,11 +4664,21 @@ mysql_execute_command(THD *thd)> TODO: fix it by removing the front element (restoring of it should> be done properly as well)> */> + /*> + If items are present in returning_list, then we need those items to point to > + INSERT table during setup_fields() and setup_wild(). But it gets masked before that. > + So we save the values in saved_first, saved_table_list and saved_first_name_resolution_context.> + before they are masked.> + */> + select_lex- >table_list.saved_first=select_lex->table_list.first;Right, this is a hack. Please check my other comment about adding saved_first.Please find a way to get the correct list of tables for name resolutionwithout extending SQL_I_List.> select_lex->table_list.first= second_table;> + select_lex- >context.saved_table_list=select_lex->context.table_list;> + selec t_lex->context.saved_name_resolution_table=> + selec t_lex->context.first_name_resolution_table;> select_lex- >context.table_list= > select_lex- >context.first_name_resolution_table= second_table;> - res= mysql_insert_select_prepare(thd);> - if (!res && (sel_result= new (thd->mem_root) select_insert(thd,> + res= mysql_insert_select_prepare(thd, lex->result ? lex->result : result);> + if (!res && (sel_result= new (thd->mem_root) select_insert(with_returning_list, lex->result ? lex->result : result, thd,> fir st_table,> first_table- >table,> &lex->field_list,> diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy> index 183a2504b70..3ef53350101 100644> --- a/sql/sql_yacc.yy> +++ b/sql/sql_yacc.yy> @@ -13333,10 +13338,15 @@ replace:> }> insert_field_spec> {> - Lex->pop_select(); //main select> - if (Lex- >check_main_unit_semantics())> - MYSQL_YYABORT;> + Lex->current_select->returning_list.swap(Lex- >current_select->item_list);> }> + opt_select_ex pressions> + {> + Lex->current_select- >returning_list.swap(Lex->current_select->item_list);> + Lex->pop_select(); //main select> + if (Lex- >check_main_unit_semantics())> + MYSQL_YYABORT;> + }Please fix indentation.> ;> > insert_lock_option:

2 2

[Maria-developers] CONC/C and CONC/ODBC API and ABI compatibility
by Michal Schorm 08 Jul '19

08 Jul '19

Hello, I am a Fedora packager and I'd like to make myself sure on some assumptions I made. Just to make sure in the following conversation: * I understand "Connector C 3.1.2" as "major version number . minor version number . release number (or sometimes refered as patch number)". * In Fedora we use dynamic libraries and dynamic linking whenever possible. We are building the CONC/ODBC on top of CONC/C. 1) ABI & API compatibility I assume, the major versions are fully API & ABI (backward) compatible. That means to me it is safe to upgrade to later versions of the same major version number when using end-user application. There's no need to rebuild the end-user app. "The MariaDB Connector/ODBC 3.1 series is built on top of MariaDB Connector/C 3.x" That means to me I can build the CONC/ODBC 3.1 on top of CONC/C 3.0 and there's no need for rebuild, when I upgrade to the CONC/C 3.1. Thus it is safe to upgrade withing one major version without any threat. That means to me I can update the package in Fedora mid release. During a lifetime of a Fedora release (let's say Fedora 30), I only update packages which won't break ABI & API compatibility. If there would be potential risk, I would push that update only to the next Fedora release (developement branch - Fedora Rawhide). That's what I do e.g. with minor versions of MariaDB server. I won't update the 10.3 to 10.4 in the lifetime of one release, but prepare the 10.4 to the next release. (Fedora Rawhide) Am I right with above assumptions? Can I safely upgrade within one major version of the C and ODBC connectors and don't need rebuild of the dependant packages and end-user apps? If not, why was CONC/ODBC 3.0 discontinued and hidden? 2) Version upgrade Is there any reason to use other than latest GA release of those connectors? (e.g. to use CONC/C 3.0 series instead of 3.1 series) Perhaps when using against different MariaDB server major versions? I assume there is not. If there is, again - why was CONC/ODBC 3.0 discontinued and hidden? -- Michal Schorm Software Engineer Core Services - Databases Team Red Hat --

3 4

[Maria-developers] Weekly Report
by Rucha Deodhar 08 Jul '19

08 Jul '19

Report for week 6: Hello! This week I was able to make replace...returning work. But before pushing changes for insert...select...returning, I had done a git merge because I didn't know I was supposed to do a git rebase. While undoing the merge I ran into a problem. With help from mentors I was able to fix the problem. I had to make some commits again. It took me two days to fix the issue and make commits and then I was on track again. I also made a GDoc about using insert...returning in subqueries. I have received feedback about my code and I have started working on fixing it. Regards, Rucha Deodhar <https://www.avast.com/en-in/recommend?utm_medium=email&utm_source=link&utm_…> I’m protected online with Avast Free Antivirus. Get it here — it’s free forever. <https://www.avast.com/en-in/recommend?utm_medium=email&utm_source=link&utm_…> <#DAB4FAD8-2DD7-40BB-A1B8-4E2AA1F9FDF2>

1 0

Re: [Maria-developers] 4670c85a48c: MDEV-16222 Assertion `0' failed in row_purge_remove_sec_if_poss_leaf on table with virtual columns and indexes
by Sergei Golubchik 05 Jul '19

05 Jul '19

Hi, Aleksey! Summary: looks ok, but please move clear_error() to the caller (ha_innodb.cc) and remove the assert. On Jul 04, Aleksey Midenkov wrote: > revision-id: 4670c85a48c (mariadb-10.4.5-47-g4670c85a48c) > parent(s): 0f55a9eb73b > author: Aleksey Midenkov <midenok(a)gmail.com> > committer: Aleksey Midenkov <midenok(a)gmail.com> > timestamp: 2019-06-27 18:05:25 +0300 > message: > > MDEV-16222 Assertion `0' failed in row_purge_remove_sec_if_poss_leaf on table with virtual columns and indexes > > Cause > Stale thd->m_stmt_da->m_sql_errno which is from different invocation. > > Fix > Reset error state before attempt to open table. > > diff --git a/mysql-test/suite/innodb/r/purge_secondary_mdev-16222.result b/mysql-test/suite/innodb/r/purge_secondary_mdev-16222.result > new file mode 100644 > index 00000000000..30e8f9800fb > --- /dev/null > +++ b/mysql-test/suite/innodb/r/purge_secondary_mdev-16222.result > @@ -0,0 +1,34 @@ ... > +select * from t1 into outfile 'load.data'; > +Warnings: > +Warning 1287 '<select expression> INTO <destination>;' is deprecated and will be removed in a future release. Please use 'SELECT <select list> INTO <destination> FROM...' instead See the warning, better not to use the deprecated syntax, unless you specifically want to test it, > +load data infile 'load.data' replace into table t1; > +set debug_sync= "now WAIT_FOR latch_released"; > +set global debug_dbug= "-d,ib_purge_virtual_mdev_16222_1"; > diff --git a/sql/sql_class.cc b/sql/sql_class.cc > index d65b6b8ed8d..cb73b7db2de 100644 > --- a/sql/sql_class.cc > +++ b/sql/sql_class.cc > @@ -4762,7 +4762,10 @@ TABLE *open_purge_table(THD *thd, const char *db, size_t dblen, > DBUG_ASSERT(!error || !ot_ctx.can_recover_from_failed_open()); > > if (unlikely(error)) > + { > close_thread_tables(thd); > + thd->clear_error(); We use thd->clear_error() in many places, but over years there were quite a few problems with it, and now I'd prefer to avoid it whenever possible. The thing is, you want to remove a specific error on a specific code path, but thd->clear_error() removes all errors, and no matter how you got here. Generally intercepting errors with Internal_error_handler is a much safer approach. In this case, though, thd->clear_error() might be okay, but better move it up the stack, do it in the caller when there can be only one way to reach this code. Not in a kind-of-generically-looking open table helper. > + } > > DBUG_RETURN(error ? NULL : tl->table); > } > diff --git a/sql/table.cc b/sql/table.cc > index 9ac167b6adb..93c2c5e88a8 100644 > --- a/sql/table.cc > +++ b/sql/table.cc > @@ -8240,6 +8240,7 @@ int TABLE::update_virtual_fields(handler *h, enum_vcol_update_mode update_mode) > > int TABLE::update_virtual_field(Field *vf) > { > + DBUG_ASSERT(!in_use->is_error()); Yes, that's what I mean above. There were cases when update_virtual_field() was called when is_error() was true and is_error() introduced a bug. I don't quite remember details, but it was something like, after the error has happened somewhere, on the way to returning an error, it was calling update_virtual_field() and some code in a function down the stack was, like some_function_which_returns_void(....); if (thd->is_error()) // the error inside a function { ... handle it ... } and the error in this case was not caused by anything in the function, but existed from before. The point is, is_error() and clear_error() are global, and you're generally interested in local state. Whether _this block_ caused an error. Clear the error caused by _this function call_. And so on, not clear an error that might've happened some time somewhere at the undefined point in the past before this code line was reached. > Query_arena backup_arena; > DBUG_ENTER("TABLE::update_virtual_field"); > in_use->set_n_backup_active_arena(expr_arena, &backup_arena); Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 2

Re: [Maria-developers] 23b70146c16: MDEV-6268 SPIDER table with no COMMENT clause causes queries to wait forever
by Sergei Golubchik 04 Jul '19

04 Jul '19

Hi, Kentoku! On Jul 04, Kentoku wrote: > revision-id: 23b70146c16 (mariadb-10.4.5-21-g23b70146c16) > parent(s): 24773bf3802 > author: Kentoku <kentokushiba(a)gmail.com> > committer: Kentoku <kentokushiba(a)gmail.com> > timestamp: 2019-06-10 13:59:18 +0900 > message: > > MDEV-6268 SPIDER table with no COMMENT clause causes queries to wait forever > Add mysql_mutex_timedlock() and add the following parameter to Spider > - spider_internal_lock_wait_timeout > The timeout when Spider tries to get internal locks. > 0 or more : the tomeout. (second) > The default value is -1 please, don't forget to leave an empty line after the first line in a commit comment. That's how various git tools (github included) separate "commit subject" from the "commit comment body". Git itself doesn't care, but it's a convention that some git-related tools rely on. Now, about the fix. Svoj made a general obervation on Slack that it's much better to eliminate or detect deadlocks, instead of wait them out with timeouts. And here we were able to find a rather cheap solution that would avoid such deadlocks in Spider. Here, how it can work: * On startup Spider calculates the unique spider instance ID by combining my_gethwaddr() and getpid(). With delimiters it might look something like "-d46d6d7345c9:23050-". This is done only once on startup. * When Spider needs to open a table, it checks whether this THD has a user variable named `spider_parents` and it it does, whether this variable contains the above mentioned unique spider instance ID. This is just one strstr(), so fast, no complex parsing. If the string is found, Spider returns an error, doesn't open a table. * When Spider connects to another host now it sends show table status from `test` like 'user' in the new approach it'll send set @spider_parents="${its current spider_parents}-d46d6d7345c9:23050-"; show table status from `test` like 'user' this is all in one packet, so no additional round-trips. Here `${its current spider_parents}` is the value of @spider_parents on the connecting host. This is it, it'll prevent self-connects, both direct and indirect via a chain of spider tables. Because @spider_parents variable is user-visible, better to use a different name for it, something that has a smaller chance of causing a conflict. May be @insternal_spider_deadlock_prevention_parent_list :) Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 1

[Maria-developers] Weekly Report
by Rucha Deodhar 01 Jul '19

01 Jul '19

Report for week 5: Hello! This week I was able to make INSERT...SELECT...RETURNING return the listed fields from the INSERT table. From the parsing stage in sql_parse.cc, the context and the first table gets masked , so I stored the previous values in another variable just before they are masked and swapped it before and after calling setup_fields() and setup_wild in select_insert :: prepare(). This makes the items and expressions in returning_list point to insert table before the above functions are called, and then swapping it back for the rest of the prepare() function. Here is the link to test cases and code: Test and result: https://github.com/MariaDB/server/commit/0792ba33a1b50e7beab3b7e05d36df29e7… Code: https://github.com/MariaDB/server/commit/4d1d37f2bdca398ff3ace804cfb3b5a940… I also tested my code for some complex queries like subquery in RETURNING and checked if it outputs error message when invalid expressions are used. It was working correctly. Here is the link for the same: Error message: https://github.com/MariaDB/server/commit/9ffe813eb82dc0125def187e07dbd5168b… Subqueries: https://github.com/MariaDB/server/commit/2363cb3993d86685ff1402cfa7ee050dd7… I was also thinking about other ways to get the items and expressions in returning_list, this is one of the ways in which it could be implemented. Here is the link to GDoc: https://docs.google.com/document/d/1TZdlL_V9MEbPTuXNwpzR0t9ciNeV25AeUIvSRXH… I understand this is too big a change for the grammar and I am trying to find alternate solution. I also changed the variable names and made them more descriptive. Regards, Rucha Deodhar

1 0

Re: [Maria-developers] [Commits] 61ad3ef: MDEV-19820 Wrong result with multiple single column index request
by Marko Mäkelä 25 Jun '19

25 Jun '19

Hi Igor, On Sun, Jun 23, 2019 at 8:26 PM IgorBabaev <igor(a)mariadb.com> wrote: > > revision-id: 61ad3efea00303e2d0cd85628c76251b08e79531 (mariadb-10.4.4-194-g61ad3ef) > parent(s): 8b576616b442d061356bc5a2abd410f478e98ee7 > author: Igor Babaev > committer: Igor Babaev > timestamp: 2019-06-23 10:26:42 -0700 > message: > > MDEV-19820 Wrong result with multiple single column index request > > The bug occured when the optimizer decided to use a rowid filter built > by a range index scan to access an InnoDB table with generated clustered > index. > When a table is accessed by a secondary index Idx employing a rowid filter the > the value of pk contained in the found index tuple is checked against the > filter. A call of the handler function position is supposed to put the > pk value into the handler::ref buffer. However for generated clustered > primary keys it did not happened. The patch fixes this problem. [snip] > diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc > index 659a824..d796b88 100644 > --- a/storage/innobase/row/row0sel.cc > +++ b/storage/innobase/row/row0sel.cc > @@ -3902,10 +3902,23 @@ row_search_idx_cond_check( > > switch (result) { > case ICP_MATCH: > - if (handler_rowid_filter_is_active(prebuilt->pk_filter) > - && !handler_rowid_filter_check(prebuilt->pk_filter)) { > - MONITOR_INC(MONITOR_ICP_MATCH); > - return(ICP_NO_MATCH); > + if (handler_rowid_filter_is_active(prebuilt->pk_filter)) { > + if (prebuilt->clust_index_was_generated) { > + ulint len; > + dict_index_t* index = prebuilt->index; > + const byte* data = rec_get_nth_field( > + rec, offsets, index->n_fields - 1, > + &len); > + ut_ad(dict_index_get_nth_col(index, > + index->n_fields - 1) > + ->prtype == (DATA_ROW_ID | DATA_NOT_NULL)); > + ut_ad(len == DATA_ROW_ID_LEN); > + memcpy(prebuilt->row_id, data, DATA_ROW_ID_LEN); > + } > + if (!handler_rowid_filter_check(prebuilt->pk_filter)) { > + MONITOR_INC(MONITOR_ICP_MATCH); > + return(ICP_NO_MATCH); > + } > } > /* Convert the remaining fields to MySQL format. > If this is a secondary index record, we must defer Here, the changed code is assuming that prebuilt->index is a secondary index without explicitly saying so. I do not see an assertion similar to ut_ad(!handler_rowid_filter_is_active(prebuilt->pk_filter) || !prebuilt->index->is_primary()); anywhere in row0sel.cc or ha_innodb.cc. A natural place for such a check could be in ha_innobase::build_template(). If the pk_filter is only being used for secondary index access, then I think that it should be enforced by some debug assertion and with a test case. Another problem is that we are doing busy work if handler_rowid_filter_check(prebuilt->pk_filter) does hold, and we will not return ICP_NO_MATCH above. I wonder if a simpler approach of reordering and deduplicating some code would work: diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 659a824a0d5..068dc849449 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -3902,11 +3902,6 @@ row_search_idx_cond_check( switch (result) { case ICP_MATCH: - if (handler_rowid_filter_is_active(prebuilt->pk_filter) - && !handler_rowid_filter_check(prebuilt->pk_filter)) { - MONITOR_INC(MONITOR_ICP_MATCH); - return(ICP_NO_MATCH); - } /* Convert the remaining fields to MySQL format. If this is a secondary index record, we must defer this until we have fetched the clustered index record. */ @@ -3920,6 +3915,10 @@ row_search_idx_cond_check( } } MONITOR_INC(MONITOR_ICP_MATCH); + if (handler_rowid_filter_is_active(prebuilt->pk_filter) + && !handler_rowid_filter_check(prebuilt->pk_filter)) { + result = ICP_NO_MATCH; + } return(result); case ICP_NO_MATCH: MONITOR_INC(MONITOR_ICP_NO_MATCH); Can you try that? With best regards, Marko

1 0

[Maria-developers] Weekly Report
by Rucha Deodhar 24 Jun '19

24 Jun '19

Report for week 4: Hello! This week I was able to make INSERT...RETURNING work. It is working fine for simple insert statement, inserting multiple rows in single statement, INSERT...SET and INSERT...ON DUPLICATE KEY UPDATE. It is returning the listed fields, working for alias and operators as well. I was getting segmentation fault when I was working on this. Zulip messages helped to solve that problem by cleaning returning_list by calling returning_list.empty() in mysql_init_select(). I also realised I hadn't called setup_fields(), setup_wild() and checking if there is '*' for returning_list in mysql_prepare_insert() which was also causing this error. Here is a link to the test cases and results for above variants of INSERT...RETURNING: https://github.com/rucha174/server/commit/66f8046d615f97a1078c8eedc3cf11d6e… As for INSERT...SELECT...RETURNING, I passed lex->field_list instead of returning_list as an argument in prepare(), send_result_set_metadata() and send_data(). It was returning all the fields from the insert table. Then I passed returning_list as the argument and it is returning the listed fields but from the select table. So now I am working on it by looking how select_insert class uses lex->field_list and do the same for returning_list. Here is the link to code which returns the listed fields for select table: https://github.com/rucha174/server/commit/6b18dd3cfc76d41871357c5b2006e04d1… Regards, Rucha Deodhar

1 0

[Maria-developers] List of the most used plugins
by GUESNET, ETIENNE (ext) 21 Jun '19

21 Jun '19

Hi, I am porting MariaDB on AIX, and some plugins do not work. As I am not a MariaDB user, I do not know which one is the most important or the most used. To prioritize correctly the port, do you have some statistic, or just a list of key plugins? I have ported for now archive, federatedX, blackhole, partition, InnoDB, aria/myisam, feedback, locales and some auth_* plugins. Thanks, Etienne.

1 0

[Maria-developers] review for MDEV-14564: Support FOR loop in stored aggregate functions
by Alexander Barkov 20 Jun '19

20 Jun '19

Hi Varun, I was recently assigned as a reviewer for this patch: http://lists.askmonty.org/pipermail/commits/2017-December/011777.html I have some suggestions. 1. The MDEV says the proposed syntax is: FOR GROUP ROWS DO and you implemented: FOR GROUP NEXT ROW DO I think the syntax on MDEV looks better. 2. I don't like that you're adding a new uint member sp_instr_agg_cfetch::m_dest An instruction processing a 'FETCH GROUP NEXT ROW' statement does not need it. 3. The generated code: +show function code f1; +Pos Instruction +0 set total at 1 0 +1 jump_if_not 5(5) 1 +2 +3 set total@1 total@1 + a@0 +4 jump 1 +5 freturn int total at 1 does not look nice. - The command at position 2 must have a name. - The command at position 1 looks like a hack. The underlying code is: + if (! it->val_bool() || thd->spcont->forced_error) So it is a conditional jump which never jumps on the condition, i.e. val_bool(). It jumps only on thd->spcont->forced_error. Please create a new class sp_instr_agg_cfetch_or_jump, derive it from sp_instr, and add m_dest to this class. Please implement print() and execute() for it. print() should display: - the name of the command, "agg_cfetch_or_jump". - the label where it jumps to when 'no rows' happens. Note, sp_instr_agg_cfetch_or_jump will have almost nothing to share with the old sp_instr_agg_cfetch. So sp_instr_agg_cfetch_or_jump should derive from sp_instr, not from sp_instr_agg_cfetch. Please perform jump to a proper position on 'no more rows' just inside sp_instr_agg_cfetch_or_jump::execute() So the code should look about like this: Pos Instruction 0 set total at 1 0 1 agg_cfetch_or_jump 4 2 set total@1 total@1 + a@0 3 jump 1 4 freturn int total at 1 4. As discussed on slack, the name for the command sp_instr_agg_cfetch was probably a not good choise. The "c" in other commands like: - sp_instr_copen - sp_instr_cfetch - sp_instr_cclose is an abbreviation for the word "cursor". sp_instr_agg_cfetch does not use any cursors (i.e. sp_cursor instances). Please rename sp_instr_agg_cfetch to sp_instr_fetch_group_row. And the new command can be named something like: sp_instr_fetch_group_row_or_jump, instead of sp_instr_agg_cfetch_or_jump. Greetings.

1 1

Re: [Maria-developers] [Commits] 8b545d3d414: MDEV-7409 On RBR, extend the PROCESSLIST info to include at least the name of the recently used table
by andrei.elkin＠pp.inet.fi 19 Jun '19

19 Jun '19

Sachin, salve. > revision-id: 8b545d3d41416772203cb98f06243873a4f0d8f7 (mariadb-10.4.5-24-g8b545d3d414) > parent(s): e35676f5557d68c7b51ba47aa73dcdf72eafa436 > author: Sachin > committer: Sachin > timestamp: 2019-05-31 12:02:37 +0530 > message: > > MDEV-7409 On RBR, extend the PROCESSLIST info to include at least the name of the recently used table > > When RBR is used, add the db name to db Field and table name to Status > Field of the "SHOW FULL PROCESSLIST" command for SQL thread. So the patch deals with the State column of the Show-Processlist command. I would endorse an output like the following: MariaDB [test]> show processlist; +----+-------------+-----------------+------+-----------+------+--------------------------------------------------+-----------------------------------------------+----------+ | Id | User | Host | db | Command | Time | State | Info | Progress | +----+-------------+-----------------+------+-----------+------+--------------------------------------------------+-----------------------------------------------+----------+ ... | 13 | system user | | test | Slave_SQL | 101 | Write_rows_log_event::write_row(-1) on table `t` | GTID 0-1-17: insert into `t` values (46),(47) | 0.000 | +----+-------------+-----------------+------+-----------+------+--------------------------------------------------+-----------------------------------------------+----------+ notice the ANSI quotes which could be more than for pretty printing considering legal names like `[:space:]+t`. Alas the text from Annotate event does not quote the table name so the actual Info is not that perfect *also* because the actual one does not display GTID.. Could we make that - the Annotate event quoted names and GTID into Progress-list Info in a separate patch for 10.5? And also to turn Progress column meaningful. It's always zero, but could print for the user the executed fraction of the whole event (sure we need to know in the Rows_log_event metadata how much rows the statement changed on the master; on slave we would just count the executed ones). > > --- > mysql-test/suite/rpl/r/rpl_rbr_monitor.result | 57 +++++++++++++++++ > mysql-test/suite/rpl/t/rpl_rbr_monitor.test | 86 +++++++++++++++++++++++++ > sql/debug_sync.cc | 3 +- > sql/log_event.cc | 90 +++++++++++++++++++++------ > 4 files changed, 215 insertions(+), 21 deletions(-) > To the test, you could really do it simpler without > --- /dev/null > +++ b/mysql-test/suite/rpl/t/rpl_rbr_monitor.test > @@ -0,0 +1,86 @@ > +--source include/have_innodb.inc > +--source include/have_debug.inc any debug. Just start a slave local trx to block Rows_log_event applying at some (see $param) point: --connection slave begin; --eval select a from t where a > $param for update; > +--source include/have_binlog_format_row.inc > +--source include/master-slave.inc > +--enable_connect_log > + > +--connection master > +create table t1(a int primary key); > +--save_master_pos > + > +--connection slave > +--sync_with_master > +SET GLOBAL debug_dbug="+d,should_wait_for_mdev7409"; > +select * from t1; > + > +--connection master > +insert into t1(a) values(1); > +--save_master_pos > + > +--echo #monitoring write rows > +--connection slave > + > +let $wait_condition= SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Write_rows_log_event::write_row(%) on table t1"; > +--source include/wait_condition.inc > +SELECT db , state FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Write_rows_log_event::write_row(%) on table t1"; > +set debug_sync="now signal cont"; > +--sync_with_master > + > +--echo #monitoring update rows > +--connection master > +update t1 set a = a + 4194304 ; > + > +--connection slave > +let $wait_condition= SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Update_rows_log_event::find_row(%) on table t1"; > +--source include/wait_condition.inc > +SELECT db, state FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Update_rows_log_event::find_row(%) on table t1"; > +set debug_sync="now signal cont1"; > + > +let $wait_condition= SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Update_rows_log_event::unpack_current_row(%) on table t1"; > +--source include/wait_condition.inc > +SELECT db, state FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Update_rows_log_event::unpack_current_row(%) on table t1"; > +set debug_sync="now signal cont2"; > + > +let $wait_condition= SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Update_rows_log_event::ha_update_row(%) on table t1"; > +--source include/wait_condition.inc > +SELECT db, state FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Update_rows_log_event::ha_update_row(%) on table t1"; > +set debug_sync="now signal cont3"; > +set debug_sync="RESET"; > +--sync_with_master > + > +--echo #monitoring delete rows > +--connection master > +delete from t1 where a>1; > + > +--connection slave > +let $wait_condition= SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Delete_rows_log_event::find_row(%) on table t1"; > +--source include/wait_condition.inc > +SELECT db , state FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Delete_rows_log_event::find_row(%) on table t1"; > +set debug_sync="now signal cont1"; > + > +let $wait_condition= SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Delete_rows_log_event::ha_delete_row(%) on table t1"; > +--source include/wait_condition.inc > +SELECT db, state FROM INFORMATION_SCHEMA.PROCESSLIST > + WHERE DB = 'test' AND STATE LIKE "Delete_rows_log_event::ha_delete_row(%) on table t1"; > +set debug_sync="now signal cont2"; > +set debug_sync="RESET"; > +--sync_with_master > +SET GLOBAL debug_dbug=""; > + > +#CleanUp > +--connection master > +drop table t1; > +--sync_slave_with_master > + > +--source include/rpl_end.inc > diff --git a/sql/debug_sync.cc b/sql/debug_sync.cc That would make all the simulation and synchronization in the source code incl the following changes unnecessary: > index bf721bddb85..7ca8d8a8ecb 100644 > --- a/sql/debug_sync.cc > +++ b/sql/debug_sync.cc > @@ -1360,7 +1360,8 @@ static void debug_sync_execute(THD *thd, st_debug_sync_action *action) > Do this before emitting the signal, so other threads can see it > if they awake before we enter_cond() below. > */ > - if (action->wait_for.length()) > + if (action->wait_for.length() && > + DBUG_EVALUATE_IF("should_wait_for_mdev7409", 0, 1)) > { > st_debug_sync_control *ds_control= thd->debug_sync_control; > strxnmov(ds_control->ds_proc_info, sizeof(ds_control->ds_proc_info)-1, > diff --git a/sql/log_event.cc b/sql/log_event.cc > index f25ebd56792..8c944a6e282 100644 > --- a/sql/log_event.cc > +++ b/sql/log_event.cc > @@ -54,6 +54,7 @@ > #include "rpl_constants.h" > #include "sql_digest.h" > #include "zlib.h" > +#include "debug_sync.h" > > #define my_b_write_string(A, B) my_b_write((A), (uchar*)(B), (uint) (sizeof(B) - 1)) > > @@ -13693,19 +13694,29 @@ Write_rows_log_event::do_exec_row(rpl_group_info *rgi) > { > DBUG_ASSERT(m_table != NULL); > const char *tmp= thd->get_proc_info(); > - const char *message= "Write_rows_log_event::write_row()"; > + LEX_CSTRING *tmp_db= &thd->db; > + char *message, msg[128]; > + my_snprintf(msg, sizeof(msg),"Write_rows_log_event::write_row() on table %s", > + m_table->s->table_name.str); > + thd->set_db(&m_table->s->db); > + message= msg; > int error; > > #ifdef WSREP_PROC_INFO > my_snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1, > - "Write_rows_log_event::write_row(%lld)", > - (long long) wsrep_thd_trx_seqno(thd)); > + "Write_rows_log_event::write_row(%lld) on table %s", > + (long long) wsrep_thd_trx_seqno(thd), m_table->s->table_name.str); > message= thd->wsrep_info; > #endif /* WSREP_PROC_INFO */ > > thd_proc_info(thd, message); > + DBUG_EXECUTE_IF("should_wait_for_mdev7409",{ > + debug_sync_set_action > + (thd, STRING_WITH_LEN("now WAIT_FOR cont")); > + };); > error= write_row(rgi, slave_exec_mode == SLAVE_EXEC_MODE_IDEMPOTENT); > thd_proc_info(thd, tmp); > + thd->set_db(tmp_db); > > if (unlikely(error) && unlikely(!thd->is_error())) > { > @@ -14368,32 +14379,47 @@ int Delete_rows_log_event::do_exec_row(rpl_group_info *rgi) > { > int error; > const char *tmp= thd->get_proc_info(); > - const char *message= "Delete_rows_log_event::find_row()"; > + LEX_CSTRING *tmp_db= &thd->db; > + char *message, msg[128]; > + my_snprintf(msg, sizeof(msg),"Delete_rows_log_event::find_row() on table %s", > + m_table->s->table_name.str); > + thd->set_db(&m_table->s->db); > + message= msg; > const bool invoke_triggers= > slave_run_triggers_for_rbr && !master_had_triggers && m_table->triggers; > DBUG_ASSERT(m_table != NULL); > > #ifdef WSREP_PROC_INFO > my_snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1, > - "Delete_rows_log_event::find_row(%lld)", > - (long long) wsrep_thd_trx_seqno(thd)); > + "Delete_rows_log_event::find_row(%lld) on table %s", > + (long long) wsrep_thd_trx_seqno(thd), m_table->s->table_name.str) ; > message= thd->wsrep_info; > #endif /* WSREP_PROC_INFO */ > > thd_proc_info(thd, message); > + DBUG_EXECUTE_IF("should_wait_for_mdev7409",{ > + debug_sync_set_action > + (thd, STRING_WITH_LEN("now WAIT_FOR cont1")); > + };); > if (likely(!(error= find_row(rgi)))) > { > /* > Delete the record found, located in record[0] > */ > - message= "Delete_rows_log_event::ha_delete_row()"; > + my_snprintf(msg, sizeof(msg),"Delete_rows_log_event::ha_delete_row() on table %s", > + m_table->s->table_name.str); > + message= msg; > #ifdef WSREP_PROC_INFO > snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1, > - "Delete_rows_log_event::ha_delete_row(%lld)", > - (long long) wsrep_thd_trx_seqno(thd)); > + "Delete_rows_log_event::ha_delete_row(%lld) on table %s", > + (long long) wsrep_thd_trx_seqno(thd), m_table->s->table_name.str) ; > message= thd->wsrep_info; > #endif > thd_proc_info(thd, message); > + DBUG_EXECUTE_IF("should_wait_for_mdev7409",{ > + debug_sync_set_action > + (thd, STRING_WITH_LEN("now WAIT_FOR cont2")); > + };); > > if (invoke_triggers && > unlikely(process_triggers(TRG_EVENT_DELETE, TRG_ACTION_BEFORE, FALSE))) > @@ -14422,6 +14448,7 @@ int Delete_rows_log_event::do_exec_row(rpl_group_info *rgi) > m_table->file->ha_index_or_rnd_end(); > } > thd_proc_info(thd, tmp); > + thd->set_db(tmp_db); > return error; > } > > @@ -14590,17 +14617,27 @@ Update_rows_log_event::do_exec_row(rpl_group_info *rgi) > const bool invoke_triggers= > slave_run_triggers_for_rbr && !master_had_triggers && m_table->triggers; > const char *tmp= thd->get_proc_info(); > - const char *message= "Update_rows_log_event::find_row()"; > + LEX_CSTRING *tmp_db= &thd->db; > + char *message, msg[128]; > DBUG_ASSERT(m_table != NULL); > + my_snprintf(msg, sizeof(msg),"Update_rows_log_event::find_row() on table %s", > + m_table->s->table_name.str); > + thd->set_db(&m_table->s->db); > + message= msg; > > #ifdef WSREP_PROC_INFO > my_snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1, > - "Update_rows_log_event::find_row(%lld)", > - (long long) wsrep_thd_trx_seqno(thd)); > + "Update_rows_log_event::find_row(%lld) on table %s", > + (long long) wsrep_thd_trx_seqno(thd), m_table->s->table_name.str) ; > message= thd->wsrep_info; > #endif /* WSREP_PROC_INFO */ > > thd_proc_info(thd, message); > + DBUG_EXECUTE_IF("should_wait_for_mdev7409",{ > + debug_sync_set_action > + (thd, STRING_WITH_LEN("now WAIT_FOR cont1")); > + };); > + > int error= find_row(rgi); > if (unlikely(error)) > { > @@ -14611,6 +14648,7 @@ Update_rows_log_event::do_exec_row(rpl_group_info *rgi) > if ((m_curr_row= m_curr_row_end)) > unpack_current_row(rgi, &m_cols_ai); > thd_proc_info(thd, tmp); > + thd->db= *tmp_db; > return error; > } > > @@ -14628,16 +14666,22 @@ Update_rows_log_event::do_exec_row(rpl_group_info *rgi) > store_record(m_table,record[1]); > > m_curr_row= m_curr_row_end; > - message= "Update_rows_log_event::unpack_current_row()"; > + my_snprintf(msg, sizeof(msg),"Update_rows_log_event::unpack_current_row() on table %s", > + m_table->s->table_name.str); > + message= msg; > #ifdef WSREP_PROC_INFO > my_snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1, > - "Update_rows_log_event::unpack_current_row(%lld)", > - (long long) wsrep_thd_trx_seqno(thd)); > + "Update_rows_log_event::unpack_current_row(%lld) on table %s", > + (long long) wsrep_thd_trx_seqno(thd), m_table->s->table_name.str) ; > message= thd->wsrep_info; > #endif /* WSREP_PROC_INFO */ > > /* this also updates m_curr_row_end */ > thd_proc_info(thd, message); > + DBUG_EXECUTE_IF("should_wait_for_mdev7409",{ > + debug_sync_set_action > + (thd, STRING_WITH_LEN("now WAIT_FOR cont2")); > + };); > if (unlikely((error= unpack_current_row(rgi, &m_cols_ai)))) > goto err; > > @@ -14655,15 +14699,21 @@ Update_rows_log_event::do_exec_row(rpl_group_info *rgi) > DBUG_DUMP("new values", m_table->record[0], m_table->s->reclength); > #endif > > - message= "Update_rows_log_event::ha_update_row()"; > + my_snprintf(msg, sizeof(msg),"Update_rows_log_event::ha_update_row() on table %s", > + m_table->s->table_name.str); > + message= msg; > #ifdef WSREP_PROC_INFO > my_snprintf(thd->wsrep_info, sizeof(thd->wsrep_info) - 1, > - "Update_rows_log_event::ha_update_row(%lld)", > - (long long) wsrep_thd_trx_seqno(thd)); > + "Update_rows_log_event::ha_update_row(%lld) on table %s", > + (long long) wsrep_thd_trx_seqno(thd), m_table->s->table_name.str) ; > message= thd->wsrep_info; > #endif /* WSREP_PROC_INFO */ > > thd_proc_info(thd, message); > + DBUG_EXECUTE_IF("should_wait_for_mdev7409",{ > + debug_sync_set_action > + (thd, STRING_WITH_LEN("now WAIT_FOR cont3")); > + };); > if (invoke_triggers && > unlikely(process_triggers(TRG_EVENT_UPDATE, TRG_ACTION_BEFORE, TRUE))) > { > @@ -14693,9 +14743,9 @@ Update_rows_log_event::do_exec_row(rpl_group_info *rgi) > unlikely(process_triggers(TRG_EVENT_UPDATE, TRG_ACTION_AFTER, TRUE))) > error= HA_ERR_GENERIC; // in case if error is not set yet > > - thd_proc_info(thd, tmp); > - > err: > + thd_proc_info(thd, tmp); > + thd->set_db(tmp_db); > m_table->file->ha_index_or_rnd_end(); > return error; > } Cheers, Andrei

2 1