- developers - lists.mariadb.org

[Maria-developers] Relevance of MDEV-19556
by Vladisav Kakurin 17 Mar '22

17 Mar '22

Hi, Vicențiu! With my mentor Nikita Malyavin we chose the topic of my bachelor's graduation work, and I was really interested in the MDEV-19556 <https://jira.mariadb.org/browse/MDEV-19556> ticket. Could you say some words about this, maybe the ticket ain’t actual, or something about work that was done by one of the developers to this direction?

1 0

Re: [Maria-developers] 2467eb2d314: MDEV-27743 Remove Lex::charset
by Sergei Golubchik 16 Mar '22

16 Mar '22

Hi, Alexander, I had only a couple of questions/comments, see below: On Mar 11, Alexander Barkov wrote: > revision-id: 2467eb2d314 (mariadb-10.6.1-330-g2467eb2d314) > parent(s): 61b72ed3dde > author: Alexander Barkov > committer: Alexander Barkov > timestamp: 2022-02-10 10:27:55 +0400 > message: > > MDEV-27743 Remove Lex::charset > > This patch also fixes: > MDEV-27690 Crash on `CHARACTER SET csname COLLATE DEFAULT` in column definition how do you plan to fix it in 10.2-10.8 ? > > diff --git a/mysql-test/main/ctype_collate_column.result b/mysql-test/main/ctype_collate_column.result > new file mode 100644 > index 00000000000..b20be5d3a48 > --- /dev/null > +++ b/mysql-test/main/ctype_collate_column.result > @@ -0,0 +1,450 @@ > +# > +# Start of 10.9 tests > +# > +# > +# MDEV-27743 Remove Lex::charset 1. Jira doesn't explain what user-visible changes one can expect. If there are none, it's not clear why you needed this big test case 2. the test is totally unreadable. Could you do a normal test instead? Like, put `select` instead of `execute immediate` and convert the output to a .test file? > +# > +CREATE TABLE cs (cs VARCHAR(64) NOT NULL); > +INSERT INTO cs (cs) VALUES > +(''), > +('CHARACTER SET latin1'), > +('CHARACTER SET utf8mb4'); > +CREATE TABLE cl0 (cl0 VARCHAR(64) NOT NULL); > +INSERT INTO cl0 (cl0) VALUES > +(''), > +('BINARY'), > +('COLLATE DEFAULT'), > +('COLLATE latin1_bin'), > +('COLLATE latin1_swedish_ci'), > +('COLLATE utf8mb4_bin'), > +('COLLATE utf8mb4_general_ci'); > +CREATE TABLE cl1 (cl1 VARCHAR(64) NOT NULL); > +INSERT INTO cl1 (cl1) VALUES > +(''), > +('COLLATE DEFAULT'), > +('COLLATE latin1_bin'), > +('COLLATE latin1_swedish_ci'), > +('COLLATE utf8mb4_bin'), > +('COLLATE utf8mb4_general_ci'); ... > diff --git a/mysql-test/main/ctype_latin1.result b/mysql-test/main/ctype_latin1.result > index 0b16d1cf6f2..70577f2c22c 100644 > --- a/mysql-test/main/ctype_latin1.result > +++ b/mysql-test/main/ctype_latin1.result > @@ -8889,6 +8889,38 @@ a b > 111 111 > DROP TABLE t1; > # > +# MDEV-27690 Crash on `CHARACTER SET csname COLLATE DEFAULT` in column definition > +# > +CREATE TABLE t1 (a CHAR(10) CHARACTER SET latin1 COLLATE DEFAULT); > +SHOW CREATE TABLE t1; > +Table Create Table > +t1 CREATE TABLE `t1` ( > + `a` char(10) DEFAULT NULL > +) ENGINE=MyISAM DEFAULT CHARSET=latin1 > +DROP TABLE t1; > +SELECT CAST('a' AS CHAR(10) CHARACTER SET latin1 COLLATE DEFAULT); > +CAST('a' AS CHAR(10) CHARACTER SET latin1 COLLATE DEFAULT) > +a > +CREATE TABLE t1 AS > +SELECT CAST('a' AS CHAR(10) CHARACTER SET latin1 COLLATE DEFAULT) AS c1; > +SHOW CREATE TABLE t1; > +Table Create Table > +t1 CREATE TABLE `t1` ( > + `c1` varchar(10) DEFAULT NULL > +) ENGINE=MyISAM DEFAULT CHARSET=latin1 > +DROP TABLE t1; > +SELECT COLUMN_GET(COLUMN_CREATE(0, 'string'),0 AS CHAR CHARACTER SET latin1 COLLATE DEFAULT) AS c1; > +c1 > +string > +CREATE TABLE t1 AS > +SELECT COLUMN_GET(COLUMN_CREATE(0, 'string'),0 AS CHAR CHARACTER SET latin1 COLLATE DEFAULT) AS c1; > +SHOW CREATE TABLE t1; > +Table Create Table > +t1 CREATE TABLE `t1` ( > + `c1` longtext DEFAULT NULL > +) ENGINE=MyISAM DEFAULT CHARSET=latin1 > +DROP TABLE t1; 1. why is it in "10.2 tests" ? 2. shouldn't here be at least one test that COLLATE DEFAULT is not a no-op? Like CREATE TABLE (... COLLATE DEFAULT ...) COLLATE _non_default ? Or is it in your generated ctype_collate_column.test? > +# > # End of 10.2 tests > # > # > diff --git a/mysql-test/suite/roles/create_and_drop_role_invalid_user_table.test b/mysql-test/suite/roles/create_and_drop_role_invalid_user_table.test > index 9cf0d7b4aff..dac6eab21e9 100644 > --- a/mysql-test/suite/roles/create_and_drop_role_invalid_user_table.test > +++ b/mysql-test/suite/roles/create_and_drop_role_invalid_user_table.test > @@ -36,7 +36,7 @@ set default role test_role for root@localhost; > drop role test_role; > drop user test_user@localhost; > > -alter table user add column default_role char(80) binary default '' not null > +alter table user add column default_role char(80) default '' not null > COLLATE utf8_general_ci Heh, interesting that it worked before good catch > after is_role; > alter table user add max_statement_time decimal(12,6) default 0 not null > diff --git a/sql/structs.h b/sql/structs.h > index 9a5006e8b47..db065c49931 100644 > --- a/sql/structs.h > +++ b/sql/structs.h > @@ -599,11 +599,140 @@ class DDL_options: public DDL_options_st ... > +class Lex_collation: public Lex_collation_st > +{ > +public: > + Lex_collation(CHARSET_INFO *collation, Type type) > + { > + m_collation= collation; > + m_type= type; > + } > + static Lex_collation national(bool bin_mod) > + { > + if (bin_mod) > + return Lex_collation(&my_charset_utf8mb3_bin, TYPE_EXPLICIT); > + return Lex_collation(&my_charset_utf8mb3_general_ci, TYPE_IMPLICIT); utf8mb3? not utf8? > + } > +}; > + > + > struct Lex_length_and_dec_st > { > -private: > +protected: > uint32 m_length; > uint8 m_dec; > + uint8 m_collation_type:2; > bool m_has_explicit_length:1; > bool m_has_explicit_dec:1; > public: > diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc > index ae6ae9a0cc0..6ca10267187 100644 > --- a/sql/sql_lex.cc > +++ b/sql/sql_lex.cc > @@ -11816,3 +11784,257 @@ bool SELECT_LEX_UNIT::explainable() const ... > +CHARSET_INFO * > +Lex_collation_st::resolved_to_character_set(CHARSET_INFO *def) const > +{ > + DBUG_ASSERT(def); > + if (m_type != TYPE_CONTEXTUALLY_TYPED) > + { > + if (!m_collation) > + return def; // Empty - not typed at all > + return m_collation; // Explicitly typed > + } > + > + // Contextually typed > + DBUG_ASSERT(m_collation); > + > + if (m_collation == &my_charset_bin) // CHAR(10) BINARY > + return find_bin_collation(def); > + > + if (m_collation == &my_charset_latin1) // CHAR(10) COLLATE DEFAULT > + return find_default_collation(def); Uhm. Could you create two dummy CHARSET_INFO variables for that instead? Like CHARSET_INFO binary_collaton, default_collation; no need to initialize them, but to have distinct values for binary/default and not reuse my_charset_bin and my_charset_latin1. > + > + /* > + Non-binary and non-default contextually typed collation. > + We don't have such yet - the parser cannot produce this. > + But will have soon, e.g. "uca1400_as_ci". > + */ > + DBUG_ASSERT(0); > + return NULL; > +} > + > + ... > diff --git a/sql/field.h b/sql/field.h > index 2ed02b37cfd..fe5ba435202 100644 > --- a/sql/field.h > +++ b/sql/field.h > @@ -5493,6 +5492,22 @@ class Column_definition: public Sql_alloc, > { return compression_method_ptr; } > > bool check_vcol_for_key(THD *thd) const; > + > + void set_lex_collation(const Lex_collation_st &lc) > + { > + charset= lc.collation(); > + if (lc.is_contextually_typed_collation()) > + flags|= BINCMP_FLAG; > + else > + flags&= ~BINCMP_FLAG; Isn't COLLATE DEFAULT also contextually typed? > + } > + Lex_collation lex_collation() const > + { > + return Lex_collation(charset, > + flags & BINCMP_FLAG ? > + Lex_collation_st::TYPE_CONTEXTUALLY_TYPED : > + Lex_collation_st::TYPE_IMPLICIT); > + } > }; > > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 2

Re: [Maria-developers] 75aa95ada6b: Tricky static asserts did not compile on some platforms. Commenting out.
by Sergei Golubchik 16 Mar '22

16 Mar '22

Hi, Alexander, On Mar 16, Alexander Barkov wrote: > revision-id: 75aa95ada6b (mariadb-10.6.1-338-g75aa95ada6b) > parent(s): e90062c20f1 > author: Alexander Barkov > committer: Alexander Barkov > timestamp: 2022-03-09 13:07:35 +0400 > message: > > Tricky static asserts did not compile on some platforms. Commenting out. Details? What doesn't compile and where? I'd suggest to try to keep the assert in some form > diff --git a/sql/structs.h b/sql/structs.h > index abfafb635b1..23958490c7b 100644 > --- a/sql/structs.h > +++ b/sql/structs.h > @@ -602,13 +602,16 @@ class DDL_options: public DDL_options_st > > struct Lex_length_and_dec_st > { > + /* > + Does not compile on some platforms: > static constexpr uint collation_type_bits= 2; > static_assert(((1<<collation_type_bits)-1) >= > Lex_charset_collation_st::TYPE_LAST); > + */ > protected: > uint32 m_length; > uint8 m_dec; > - uint8 m_collation_type:collation_type_bits; > + uint8 m_collation_type:2; /*collation_type_bits;*/ > bool m_has_explicit_length:1; > bool m_has_explicit_dec:1; > public: Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] e90062c20f1: MDEV-27266 Improve UCA collation performance for utf8mb3 and utf8mb4
by Sergei Golubchik 16 Mar '22

16 Mar '22

Hi, Alexander, also ok On Mar 14, Alexander Barkov wrote: > revision-id: e90062c20f1 (mariadb-10.6.1-337-ge90062c20f1) > parent(s): 6a3201d3769 > author: Alexander Barkov > committer: Alexander Barkov > timestamp: 2022-02-28 14:43:43 +0400 > message: > > MDEV-27266 Improve UCA collation performance for utf8mb3 and utf8mb4 > > Adding two levels of optimization: > > include/m_ctype.h | 53 ++++ > strings/ctype-uca-scanner_next.inl | 39 +++ > strings/ctype-uca.c | 548 ++++++++++++++++++++++++++++++++++++- > strings/ctype-uca.ic | 30 ++ I thought all *.ic files were renamed to *.inl ? > unittest/strings/strings-t.c | 2 +- > 5 files changed, 660 insertions(+), 12 deletions(-) Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 6a3201d3769: MDEV-27265 Improve contraction performance in UCA collations
by Sergei Golubchik 16 Mar '22

16 Mar '22

Hi, Alexander, that ok, up to you. I'm not looking into the strings/ctype*c code. On Mar 14, Alexander Barkov wrote: > revision-id: 6a3201d3769 (mariadb-10.6.1-336-g6a3201d3769) > parent(s): 49ecf935415 > author: Alexander Barkov > committer: Alexander Barkov > timestamp: 2022-02-28 14:05:08 +0400 > message: > > MDEV-27265 Improve contraction performance in UCA collations > > Adding a hash table for contractions. > > The old code iterated through all items in MY_CONTRACTIONS, > and was much slower, especially for those contractions > in the end of the list. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] c67789f63c8: MDEV-27009 Add UCA-14.0.0 collations - adding version aware implicit weight handling
by Sergei Golubchik 14 Mar '22

14 Mar '22

Hi, Alexander, this is ok too, thanks On Mar 14, Alexander Barkov wrote: > revision-id: c67789f63c8 (mariadb-10.6.1-334-gc67789f63c8) > parent(s): c1010fdfac2 > author: Alexander Barkov > committer: Alexander Barkov > timestamp: 2022-02-28 10:31:31 +0400 > message: > > MDEV-27009 Add UCA-14.0.0 collations - adding version aware implicit weight handling > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] c1010fdfac2: MDEV-27009 Add UCA-14.0.0 collations - dump logical positions and contractions
by Sergei Golubchik 14 Mar '22

14 Mar '22

Hi, Alexander, this is ok too On Mar 14, Alexander Barkov wrote: > revision-id: c1010fdfac2 (mariadb-10.6.1-333-gc1010fdfac2) > parent(s): ab62059bbd4 > author: Alexander Barkov > committer: Alexander Barkov > timestamp: 2022-02-28 10:31:31 +0400 > message: > > MDEV-27009 Add UCA-14.0.0 collations - dump logical positions and contractions > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] ab62059bbd4: MDEV-27009 Add UCA-14.0.0 collations - Adding implicit weight handling for Unicode-14.0.0
by Sergei Golubchik 14 Mar '22

14 Mar '22

Hi, Alexander, I believe it's ok too, but see a couple of questions below On Mar 14, Alexander Barkov wrote: > revision-id: ab62059bbd4 (mariadb-10.6.1-332-gab62059bbd4) > parent(s): 5fe33342399 > author: Alexander Barkov > committer: Alexander Barkov > timestamp: 2022-02-28 10:31:06 +0400 > message: > > MDEV-27009 Add UCA-14.0.0 collations - Adding implicit weight handling for Unicode-14.0.0 > > diff --git a/strings/ctype-uca.c b/strings/ctype-uca.c > index ad0b46ea296..eb6bc2eb9e6 100644 > --- a/strings/ctype-uca.c > +++ b/strings/ctype-uca.c > @@ -31694,7 +31694,7 @@ static inline void > my_uca_implicit_weight_put(uint16 *to, my_wc_t code, uint level) > { > MY_UCA_IMPLICIT_WEIGHT weight; > - weight= my_uca_520_implicit_weight_on_level(code, level); > + weight= my_uca_implicit_weight_on_level(520, code, level); will the version always be a constant in every invocation of my_uca_implicit_weight_put() ? > to[0]= weight.weight[0]; > to[1]= weight.weight[1]; > to[2]= 0; > diff --git a/strings/ctype-uca.h b/strings/ctype-uca.h > index 01f655b06cb..d439b8c578c 100644 > --- a/strings/ctype-uca.h > +++ b/strings/ctype-uca.h > @@ -136,11 +149,11 @@ my_uca_implicit_weight_quaternary() > > > static inline MY_UCA_IMPLICIT_WEIGHT > -my_uca_520_implicit_weight_on_level(my_wc_t code, uint level) > +my_uca_implicit_weight_on_level(uint version, my_wc_t code, uint level) > { > switch (level) { > case 0: > - return my_uca_520_implicit_weight_primary(code); > + return my_uca_implicit_weight_primary(version, code); > case 1: > return my_uca_implicit_weight_secondary(); > case 2: do you mean that secondary/tertiary/etc weights do not depend on the version? Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 5fe33342399: MDEV-27009 Add UCA-14.0.0 collations - adding uca-dump into build targets
by Sergei Golubchik 14 Mar '22

14 Mar '22

Hi, Alexander, this is ok (not ok to push as is just yet, please wait until we agree on commits in the branch) On Mar 14, Alexander Barkov wrote: > revision-id: 5fe33342399 (mariadb-10.6.1-331-g5fe33342399) > parent(s): 2467eb2d314 > author: Alexander Barkov > committer: Alexander Barkov > timestamp: 2022-02-10 11:53:47 +0400 > message: > > MDEV-27009 Add UCA-14.0.0 collations - adding uca-dump into build targets > > - Adding uca-dump into build targets > - Adding ctype-uca.h and moving implicit weight related routines there > - Reusing implicit weight routines in ctype-uca.c and uca-dump.c > - Adding handling of command line arguments to uca-dump > - Fixing some compile-time warnings in uca-dump.c > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 61b72ed3dde: MDEV-27712 Reduce the size of Lex_length_and_dec_st from 16 to 8
by Sergei Golubchik 11 Mar '22

11 Mar '22

Hi, Alexander, a question about error codes: On Mar 11, Alexander Barkov wrote: > commit 61b72ed3dde > Author: Alexander Barkov <bar(a)mariadb.com> > Date: Wed Feb 9 17:59:38 2022 +0400 > > diff --git a/mysql-test/main/type_blob.test b/mysql-test/main/type_blob.test > index c61ed124139..a5410244ed1 100644 > --- a/mysql-test/main/type_blob.test > +++ b/mysql-test/main/type_blob.test > @@ -517,7 +517,7 @@ CREATE TABLE b15776 (a char(2147483648)); > --error ER_TOO_BIG_FIELDLENGTH > CREATE TABLE b15776 (a char(4294967295)); > # Even BLOB won't hold > ---error ER_TOO_BIG_FIELDLENGTH > +--error ER_NOT_SUPPORTED_YET Why did you change the error? > CREATE TABLE b15776 (a char(4294967296)); > > > diff --git a/mysql-test/main/type_float.result b/mysql-test/main/type_float.result > index 5137a8229b6..9e216711ddd 100644 > --- a/mysql-test/main/type_float.result > +++ b/mysql-test/main/type_float.result > @@ -1162,3 +1162,36 @@ fdec 123.456.789,12345678900000000000000000000000000000 > # > # End of 10.4 tests > # > +# > +# Start of 10.9 tests > +# > +# > +# MDEV-27712 Reduce the size of Lex_length_and_dec_st from 16 to 8 > +# > +CREATE TABLE t1 (a DOUBLE(1000,1000)); > +ERROR 42000: This version of MariaDB doesn't yet support 'scale>255' > +CREATE TABLE t1 (a DOUBLE(1000,0)); > +ERROR 42000: Display width out of range for 'a' (max = 255) > +CREATE TABLE t1 (a DOUBLE(0,1000)); > +ERROR 42000: This version of MariaDB doesn't yet support 'scale>255' > +CREATE TABLE t1 (a DOUBLE(2147483647,2147483647)); > +ERROR 42000: This version of MariaDB doesn't yet support 'scale>255' > +CREATE TABLE t1 (a DOUBLE(2147483647,0)); > +ERROR 42000: Display width out of range for 'a' (max = 255) > +CREATE TABLE t1 (a DOUBLE(0,2147483647)); > +ERROR 42000: This version of MariaDB doesn't yet support 'scale>255' > +CREATE TABLE t1 (a DOUBLE(2147483648,2147483648)); > +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '2147483648,2147483648))' at line 1 This isn't technically a *syntax* error Can you use a different error here? > +CREATE TABLE t1 (a DOUBLE(2147483648,0)); > +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '2147483648,0))' at line 1 > +CREATE TABLE t1 (a DOUBLE(0,2147483648)); > +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '2147483648))' at line 1 > +CREATE TABLE t1 (a DOUBLE(999999999999999999999999999999,999999999999999999999999999999)); > +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '999999999999999999999999999999,999999999999999999999999999999))' at line 1 > +CREATE TABLE t1 (a DOUBLE(999999999999999999999999999999,0)); > +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '999999999999999999999999999999,0))' at line 1 > +CREATE TABLE t1 (a DOUBLE(0,999999999999999999999999999999)); > +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '999999999999999999999999999999))' at line 1 > +# > +# End of 10.9 tests > +# > diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy > index 5bb03544762..fdc54f7df27 100644 > --- a/sql/sql_yacc.yy > +++ b/sql/sql_yacc.yy > @@ -5975,11 +5977,10 @@ field_type_numeric: > | FLOAT_SYM float_options last_field_options > { > $$.set(&type_handler_float, $2); > - if ($2.length() && !$2.dec()) > + if ($2.has_explicit_length() && !$2.has_explicit_dec()) > { > - int err; > - ulonglong tmp_length= my_strtoll10($2.length(), NULL, &err); > - if (unlikely(err || tmp_length > PRECISION_FOR_DOUBLE)) > + ulonglong tmp_length= $2.length(); you don't really need tmp_length here anymore > + if (unlikely(tmp_length > PRECISION_FOR_DOUBLE)) > my_yyabort_error((ER_WRONG_FIELD_SPEC, MYF(0), > Lex->last_field->field_name.str)); > if (tmp_length > PRECISION_FOR_FLOAT) Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 0402f6dcb67: MDEV-25477 Auto-create breaks replication when triggering event was not replicated
by Sergei Golubchik 07 Mar '22

07 Mar '22

Hi, Aleksey, This was quite good, thanks! Many cases covered, not just ROLLBACK, but also LIMIT 0 and impossible WHERE. See a couple of comments below. Nothing big. On Feb 12, Aleksey Midenkov wrote: > commit 0402f6dcb67 > Author: Aleksey Midenkov <midenok(a)gmail.com> > Date: Tue Feb 8 22:54:03 2022 +0300 > > diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc > index 0403d6e73c3..06068290247 100644 > --- a/sql/sql_delete.cc > +++ b/sql/sql_delete.cc > @@ -507,6 +508,18 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, > if (thd->lex->describe || thd->lex->analyze_stmt) > goto produce_explain_and_leave; > > + if (thd->log_current_statement) > + { > + thd->reset_unsafe_warnings(); > + if (thd->binlog_query(THD::STMT_QUERY_TYPE, > + thd->query(), thd->query_length(), > + transactional_table, FALSE, FALSE, 0) > 0) > + { > + my_error(ER_ERROR_ON_WRITE, MYF(0), "binary log", -1); > + DBUG_RETURN(1); > + } > + } perhaps, move this block into a function? It's repeated four times here. by the way, you don't seem to have a test for a zero-rows insert. That is INSERT ... SELECT with an impossible where or limit 0. In fact, you can have a zero-row insert with a normal insert INSERT t1 VALUES (1); if this (1) is a unique key violation. I believe your code handles this case, thought, just a test case is missing. > + > my_ok(thd, 0); > DBUG_RETURN(0); > } > @@ -932,8 +958,8 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, > else > errcode= query_error_code(thd, killed_status == NOT_KILLED); > > - ScopedStatementReplication scoped_stmt_rpl( > - table->versioned(VERS_TRX_ID) ? thd : NULL); > + StatementBinlog stmt_binlog(thd, table->versioned(VERS_TRX_ID) || > + thd->binlog_need_stmt_format(transactional_table)); I know that's not part of this patch, but still. Just trying to understand. Why did it change to statement-based if VERS_TRX_ID? > /* > [binlog]: If 'handler::delete_all_rows()' was called and the > storage engine does not inject the rows itself, we replicate Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 3

Re: [Maria-developers] 880e92a48ba: MDEV-27760 event may non stop replicate in circular semisync setup
by Sergei Golubchik 07 Mar '22

07 Mar '22

Hi, Andrei, I don't see a point in caching the value in memory that is very cheap to calculate per event. I've pushed f8ec4dd38d0 to 10.4-serg three hours ago and Jenkins said it's fine. So I'd suggest we go with it, because we really have to close this blocker and release ES now. And MDEV-27837 will be fixed as a separate bug. queue_event checks global_system_variables.server_id in multiple places, moving only do_accept_own_server_id into Master_info will not make it any better or worse against MDEV-27837. > > As far as I can see, you can calculate it for every event just the same. > > The per event computation is extraneous and not consistent with the > nature of the flag that belongs to the group of events. > > Also notice (or remember) a "FR" > MDEV-27837 disallow `set @@session.server_id` within transaction > so until this bug is fixed it's not 'just the same'. > > I am recomming the patch, assuming that you'll be fine with the Gtid time > only flag computation. > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 880e92a48ba: MDEV-27760 event may non stop replicate in circular semisync setup
by Sergei Golubchik 06 Mar '22

06 Mar '22

Hi, Andrei, On Mar 01, Andrei Elkin wrote: > > On Feb 28, Andrei wrote: > >> revision-id: 880e92a48ba (mariadb-10.6.6-16-g880e92a48ba) > >> parent(s): 4030a9fb2eb > >> author: Andrei > >> committer: Andrei > >> timestamp: 2022-02-21 14:58:13 +0200 > >> message: > >> > >> MDEV-27760 event may non stop replicate in circular semisync setup > >> > >> diff --git a/sql/slave.cc b/sql/slave.cc > >> index c0eef02ca7a..a1f4f4f0081 100644 > >> --- a/sql/slave.cc > >> +++ b/sql/slave.cc > >> @@ -6186,13 +6186,13 @@ static int queue_event(Master_info* mi, const uchar *buf, ulong event_len) > >> bool is_rows_event= false; > >> /* > >> The flag has replicate_same_server_id semantics and is raised to accept > >> - a same-server-id event on the semisync slave, for both the gtid and legacy > >> - connection modes. > >> - Such events can appear as result of this server recovery so the event > >> - was created there and replicated elsewhere right before the crash. At recovery > >> - it could be evicted from the server's binlog. > >> - */ > >> - bool do_accept_own_server_id= false; > >> + a same-server-id event group by the gtid strict mode semisync slave. > >> + Own server-id events can appear as result of this server crash-recovery: > >> + the transaction was created on this server then being master, got replicated > >> + elsewhere right before the crash before commit; > >> + finally at recovery the transaction gets evicted from the server's binlog. > >> + */ > >> + static bool do_accept_own_server_id= false; > > > > No, sorry, it cannot be static. If you need to preserve it between > > events, you can keep it, for example, in Master_info. > > The var is set by Gtid events to serve as a marker of the whole current Gtid-headed > group of event. > Only next Gtid event may change the current value. > > Why can't it? Why it cannot be static? Because it's not a property of the server, it's a property of a specific replication stream. Technically one can have a multi-source and do_accept_own_server_id could have different values for different masters. > > So, the question now is, why do you need to preserve the value of > > do_accept_own_server_id between queue_event invocations? > > The flag as said above applies to the whole group of events, so it's > preserved at least through so many > queue_event() calls as many event in the group (I count Gtid itself in). As far as I can see, you can calculate it for every event just the same. > >> @@ -6281,6 +6281,8 @@ static int queue_event(Master_info* mi, const uchar *buf, ulong event_len) > >> dbug_rows_event_count = 0; > >> };); > >> #endif > >> + s_id= uint4korr(buf + SERVER_ID_OFFSET); > >> + > >> mysql_mutex_lock(&mi->data_lock); > >> > >> switch (buf[EVENT_TYPE_OFFSET]) { > >> @@ -6722,6 +6724,11 @@ static int queue_event(Master_info* mi, const uchar *buf, ulong event_len) > >> > >> ++mi->events_queued_since_last_gtid; > >> inc_pos= event_len; > >> + > >> + do_accept_own_server_id= > >> + (s_id == global_system_variables.server_id && rpl_semi_sync_slave_enabled > >> + && mi->using_gtid != Master_info::USE_GTID_NO && opt_gtid_strict_mode) ? > >> + true : false; Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

[Maria-developers] MDEV-27776 EXPLAIN is about to get new columns, EXPLAIN EXTENDED|PARTITIONS will be deprecated
by Oleg Smirnov 03 Mar '22

03 Mar '22

* EXPLAIN SELECT prints "partitions", "filtered" columns and produces the warning with rewritten query * EXPLAIN EXTENDED and EXPLAIN PARTITIONS produce the same output as EXPLAIN SELECT and two warnings, the second warning is about EXPLAIN EXTENDED| PARTITIONS being deprecated (same as in MySQL 5.7) -- Best regards, ------------------------- *Oleg Smirnov* Sr. Software Engineer MariaDB Server Team

1 0

Re: [Maria-developers] 5d8838a79d5: MDEV-24920: Merge "old" SQL variable to "old_mode" sql variable
by Sergei Golubchik 02 Mar '22

02 Mar '22

Hi, Rucha, On Mar 02, Rucha Deodhar wrote: > revision-id: 5d8838a79d5 (mariadb-10.6.1-352-g5d8838a79d5) > parent(s): f8b3c661231 > author: Rucha Deodhar > committer: Rucha Deodhar > timestamp: 2022-02-15 22:29:54 +0530 > message: > > MDEV-24920: Merge "old" SQL variable to "old_mode" sql variable > > Analysis: There are 2 server variables- "old_mode" and "old". "old" is no > longer needed as "old_mode" has replaced it (however still used in some places > in the code). "old_mode" and "old" has same purpose- emulate behavior from > previous MariaDB versions. So they can be merged to avoid confusion. > Fix: Deprecate "old" variable and create another mode for @@old_mode to mimic > behavior of previous "old" variable. Create specific modes for specifix task > that --old sql variable was doing earlier and use the new modes instead. > > diff --git a/mysql-test/main/mysqld--help.result b/mysql-test/main/mysqld--help.result > index 7cbfa52b846..261dbe2ffe0 100644 > --- a/mysql-test/main/mysqld--help.result > +++ b/mysql-test/main/mysqld--help.result > @@ -683,7 +683,8 @@ The following specify which files/extra groups are read (specified before remain > --old-mode=name Used to emulate old behavior from earlier MariaDB or > MySQL versions. Any combination of: > NO_DUP_KEY_WARNINGS_WITH_IGNORE, NO_PROGRESS_INFO, > - ZERO_DATE_TIME_CAST, UTF8_IS_UTF8MB3 > + ZERO_DATE_TIME_CAST, UTF8_IS_UTF8MB3, > + INDEX_HINT_MASK_JOIN, CHECKSUM_SLOW_NULLS I'm sorry, I still cannot understand what these new modes do :( I know that it was my idea to use "CHECKSUM_SLOW_NULLS", but now I'm, like, "what does it mean?". Let's discuss it on slack INDEX_HINT_MASK_JOIN - this seems to be easier, what about INGORE_INDEX_ONLY_FOR_JOIN ? That's a bit long, but shorter than NO_DUP_KEY_WARNINGS_WITH_IGNORE. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 880e92a48ba: MDEV-27760 event may non stop replicate in circular semisync setup
by Sergei Golubchik 01 Mar '22

01 Mar '22

Hi, Andrei, On Feb 28, Andrei wrote: > revision-id: 880e92a48ba (mariadb-10.6.6-16-g880e92a48ba) > parent(s): 4030a9fb2eb > author: Andrei > committer: Andrei > timestamp: 2022-02-21 14:58:13 +0200 > message: > > MDEV-27760 event may non stop replicate in circular semisync setup > > diff --git a/sql/slave.cc b/sql/slave.cc > index c0eef02ca7a..a1f4f4f0081 100644 > --- a/sql/slave.cc > +++ b/sql/slave.cc > @@ -6186,13 +6186,13 @@ static int queue_event(Master_info* mi, const uchar *buf, ulong event_len) > bool is_rows_event= false; > /* > The flag has replicate_same_server_id semantics and is raised to accept > - a same-server-id event on the semisync slave, for both the gtid and legacy > - connection modes. > - Such events can appear as result of this server recovery so the event > - was created there and replicated elsewhere right before the crash. At recovery > - it could be evicted from the server's binlog. > - */ > - bool do_accept_own_server_id= false; > + a same-server-id event group by the gtid strict mode semisync slave. > + Own server-id events can appear as result of this server crash-recovery: > + the transaction was created on this server then being master, got replicated > + elsewhere right before the crash before commit; > + finally at recovery the transaction gets evicted from the server's binlog. > + */ > + static bool do_accept_own_server_id= false; No, sorry, it cannot be static. If you need to preserve it between events, you can keep it, for example, in Master_info. yes, I remember, that's where it was and I asked why it's in Master_info. You could've answered, that you need to preserve the value between queue_event invocations - I didn't realize that at the time. So, the question now is, why do you need to preserve the value of do_accept_own_server_id between queue_event invocations? > /* > FD_q must have been prepared for the first R_a event > inside get_master_version_and_clock() > @@ -6281,6 +6281,8 @@ static int queue_event(Master_info* mi, const uchar *buf, ulong event_len) > dbug_rows_event_count = 0; > };); > #endif > + s_id= uint4korr(buf + SERVER_ID_OFFSET); > + > mysql_mutex_lock(&mi->data_lock); > > switch (buf[EVENT_TYPE_OFFSET]) { > @@ -6722,6 +6724,11 @@ static int queue_event(Master_info* mi, const uchar *buf, ulong event_len) > > ++mi->events_queued_since_last_gtid; > inc_pos= event_len; > + > + do_accept_own_server_id= > + (s_id == global_system_variables.server_id && rpl_semi_sync_slave_enabled > + && mi->using_gtid != Master_info::USE_GTID_NO && opt_gtid_strict_mode) ? > + true : false; this is a ridiculous pattern bool var = (bool expr) ? true : false; don't do that. > } > break; > /* > @@ -6909,7 +6916,6 @@ static int queue_event(Master_info* mi, const uchar *buf, ulong event_len) > */ > > mysql_mutex_lock(log_lock); > - s_id= uint4korr(buf + SERVER_ID_OFFSET); > /* > Write the event to the relay log, unless we reconnected in the middle > of an event group and now need to skip the initial part of the group that > @@ -6955,7 +6961,7 @@ static int queue_event(Master_info* mi, const uchar *buf, ulong event_len) > else > if ((s_id == global_system_variables.server_id && > !(mi->rli.replicate_same_server_id || > - (do_accept_own_server_id= rpl_semi_sync_slave_enabled))) || > + do_accept_own_server_id)) || > event_that_should_be_ignored(buf) || > /* > the following conjunction deals with IGNORE_SERVER_IDS, if set > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] MDEV-16329 ALTER ONLINE TABLE
by Sergei Golubchik 27 Feb '22

27 Feb '22

Hi, Nikita, this is a review of the combined diff of your branch. Of `git diff github/10.8...github/bb-10.8-online-alter`, that is of cf7cc376bac fix savepoints in myisam 4a128ca5458 fix main.delayed 7300151cc84 fix skipping rocksdb 19504dfdbcc add binlog/standalone combinations 55e993595aa add rocksdb combination 06645b17450 savepoints 9ffcd13b9fe rename tests 730926ae962 MDEV-16329 [5/5] ALTER ONLINE TABLE cf52ea33a70 MDEV-16329 [4/5] Refactor MYSQL_BIN_LOG: extract Event_log ancestor 0445c8f1ee9 MDEV-16329 [3/5] use binlog_cache_data directly in most places 3f3da0f65a3 MDEV-16329 [2/5] refactor binlog and cache_mngr dc79667ca73 MDEV-16329 [1/5] add THD::binlog_get_cache_mngr 6f9dd00ce2d rpl: repack table_def faf8680df8d Copy_field: add const to arguments It was mostly fine the last time already, so only few comments below: > diff --git a/mysql-test/include/have_log_bin_off.require b/mysql-test/include/have_log_bin_off.require > new file mode 100644 > index 00000000000..979dbe75f80 > --- /dev/null > +++ b/mysql-test/include/have_log_bin_off.require > @@ -0,0 +1,2 @@ > +Variable_name Value > +log_bin OFF better use `if (...) { skip }` pattern *.require is an obsolete 20-year old technique from before if- and skip-times > diff --git a/mysql-test/main/alter_table.test b/mysql-test/main/alter_table.test > index 31c69783248..91423ee8c1f 100644 > --- a/mysql-test/main/alter_table.test > +++ b/mysql-test/main/alter_table.test > @@ -1532,8 +1532,12 @@ ALTER TABLE t1 DROP INDEX i1, DROP INDEX i2, DROP INDEX i3, DROP INDEX i4; > ALTER TABLE t1 ADD INDEX i1(b), ALGORITHM= INPLACE, LOCK= NONE; > ALTER TABLE t1 ADD INDEX i2(b), ALGORITHM= INPLACE, LOCK= SHARED; > ALTER TABLE t1 ADD INDEX i3(b), ALGORITHM= INPLACE, LOCK= EXCLUSIVE; > ---error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON > +--disable_info > +--disable_warnings > +--error 0,ER_ALTER_OPERATION_NOT_SUPPORTED_REASON why, because of embedded? > ALTER TABLE t1 ADD INDEX i4(b), ALGORITHM= COPY, LOCK= NONE; > +--enable_warnings > +--enable_info > ALTER TABLE t1 ADD INDEX i5(b), ALGORITHM= COPY, LOCK= SHARED; > ALTER TABLE t1 ADD INDEX i6(b), ALGORITHM= COPY, LOCK= EXCLUSIVE; > > diff --git a/mysql-test/main/alter_table_locknone.result b/mysql-test/main/alter_table_locknone.result > new file mode 100644 > index 00000000000..ea040925efe > --- /dev/null > +++ b/mysql-test/main/alter_table_locknone.result > @@ -0,0 +1,270 @@ > +create table t1 (a int not null primary key, b int, c varchar(80), e enum('a','b')) engine=myisam; > +insert into t1 (a) values (1),(2),(3); > +alter online table t1 modify b int default 5, alter c set default 'X'; > +alter online table t1 change b new_name int; > +alter online table t1 modify e enum('a','b','c'); > +alter online table t1 comment "new comment"; > +alter table t1 add constraint q check (a > 0); > +alter online table t1 drop constraint q; > +alter online table t1 algorithm=INPLACE, lock=NONE; > +alter online table t1; > +alter table t1 algorithm=INPLACE; > +alter table t1 lock=NONE; > +show create table t1; > +Table Create Table > +t1 CREATE TABLE `t1` ( > + `a` int(11) NOT NULL, > + `new_name` int(11) DEFAULT NULL, > + `c` varchar(80) DEFAULT 'X', > + `e` enum('a','b','c') DEFAULT NULL, > + PRIMARY KEY (`a`) > +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COMMENT='new comment' > +drop table t1; > +create temporary table t1 (a int not null primary key, b int, c varchar(80), e enum('a','b')); > +insert into t1 (a) values (1),(2),(3); > +alter online table t1 modify b int default 5, alter c set default 'X'; > +alter online table t1 change b new_name int; > +alter online table t1 modify e enum('a','b','c'); > +alter online table t1 comment "new comment"; > +alter online table t1 rename to t2; > +show create table t2; > +Table Create Table > +t2 CREATE TEMPORARY TABLE `t2` ( > + `a` int(11) NOT NULL, > + `new_name` int(11) DEFAULT NULL, > + `c` varchar(80) DEFAULT 'X', > + `e` enum('a','b','c') DEFAULT NULL, > + PRIMARY KEY (`a`) > +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COMMENT='new comment' > +drop table t2; > +create table t1 (a int not null primary key, b int, c varchar(80), e enum('a','b')) engine=aria; > +insert into t1 (a) values (1),(2),(3); > +alter online table t1 modify b int default 5; > +alter online table t1 change b new_name int; > +alter online table t1 modify e enum('a','b','c'); > +alter online table t1 comment "new comment"; > +show create table t1; > +Table Create Table > +t1 CREATE TABLE `t1` ( > + `a` int(11) NOT NULL, > + `new_name` int(11) DEFAULT NULL, > + `c` varchar(80) DEFAULT NULL, > + `e` enum('a','b','c') DEFAULT NULL, > + PRIMARY KEY (`a`) > +) ENGINE=Aria DEFAULT CHARSET=latin1 PAGE_CHECKSUM=1 COMMENT='new comment' > +alter online table t1 page_checksum=1; > +alter online table t1 page_checksum=0; > +ERROR 0A000: LOCK=NONE is not supported for this operation. Try LOCK=SHARED Why LOCK=NONE would be not supported now? Shouldn't your new feature allow just everything with LOCK=NONE? > +drop table t1; > +create table t1 (a int not null primary key, b int, c varchar(80), e enum('a','b')); > +insert into t1 (a) values (1),(2),(3); > +alter online table t1 drop column b, add b int; > +ERROR 0A000: LOCK=NONE is not supported for this operation. Try LOCK=SHARED ... > diff --git a/mysql-test/main/alter_table_online.combinations b/mysql-test/main/alter_table_online.combinations > new file mode 100644 > index 00000000000..ae144432c68 > --- /dev/null > +++ b/mysql-test/main/alter_table_online.combinations > @@ -0,0 +1,2 @@ > +[innodb] > +[rocksdb] what about non-trans tables? > diff --git a/mysql-test/main/alter_table_online.test b/mysql-test/main/alter_table_online.test > index 6ef9661c43b..815483d252d 100644 > --- a/mysql-test/main/alter_table_online.test > +++ b/mysql-test/main/alter_table_online.test > @@ -1,335 +1,596 @@ > -# > -# Test of ALTER ONLINE TABLE syntax > -# > - > ---source include/have_innodb.inc > ---source include/have_partition.inc > -# > -# Test of things that can be done online > -# > - > -create table t1 (a int not null primary key, b int, c varchar(80), e enum('a','b')) engine=myisam; > -insert into t1 (a) values (1),(2),(3); > - > -alter online table t1 modify b int default 5, alter c set default 'X'; > -alter online table t1 change b new_name int; > -alter online table t1 modify e enum('a','b','c'); > -alter online table t1 comment "new comment"; > -alter table t1 add constraint q check (a > 0); > -alter online table t1 drop constraint q; > - > -# No OPs > - > -alter online table t1 algorithm=INPLACE, lock=NONE; > -alter online table t1; > -alter table t1 algorithm=INPLACE; > -alter table t1 lock=NONE; > -show create table t1; > -drop table t1; > +-- source include/have_debug_sync.inc > +-- source include/not_embedded.inc > +-- source alter_table_online_binlog.inc > > -# > -# everything with temporary tables is "online", i.e. without locks > -# > -create temporary table t1 (a int not null primary key, b int, c varchar(80), e enum('a','b')); > -insert into t1 (a) values (1),(2),(3); > - > -alter online table t1 modify b int default 5, alter c set default 'X'; > -alter online table t1 change b new_name int; > -alter online table t1 modify e enum('a','b','c'); > -alter online table t1 comment "new comment"; > -alter online table t1 rename to t2; > -show create table t2; > -drop table t2; > +-- disable_query_log > +-- if ($MTR_COMBINATION_INNODB) { > +-- source include/have_innodb.inc > +set default_storage_engine= innodb; > +-- } > +-- if ($MTR_COMBINATION_ROCKSDB) { > +-- source include/have_rocksdb.inc > +set default_storage_engine= rocksdb; > +-- } > +-- enable_query_log > + > +-- let $default_engine= `select @@default_storage_engine` > + > +--connect (con2, localhost, root,,) > +--connection default > + > + > +--echo # > +--echo # Test insert > +--echo # > + > +--echo # Insert and add column > +create or replace table t1 (a int) engine=innodb; why do you specify the engine explicitly if you run it in two engine combinations? > +insert t1 values (5); > + > +--connection con2 > +--send > +set debug_sync= 'now WAIT_FOR ended'; > + > +--connection default > +set debug_sync= 'alter_table_copy_end SIGNAL ended WAIT_FOR end'; > + ... > + > +--connection default > +--reap > +--sorted_result > +select * from t1; > + > +--echo # > +--echo # MYISAM. Only Inserts can be tested. why? > +--echo # > + > +create or replace table t1 (a int) engine=myisam; > +insert t1 values (5); > + > +--connection con2 > +--send > +set debug_sync= 'now WAIT_FOR ended'; > + > +--connection default > +set debug_sync= 'alter_table_copy_end SIGNAL ended WAIT_FOR end'; > + > +--send > +alter table t1 add b int NULL, algorithm= copy, lock= none; > + ... > diff --git a/mysql-test/main/mdl_sync.test b/mysql-test/main/mdl_sync.test > index 3df19aca806..8c6bc76f5ca 100644 > --- a/mysql-test/main/mdl_sync.test > +++ b/mysql-test/main/mdl_sync.test > @@ -42,7 +42,7 @@ connection con1; > set debug_sync='mdl_upgrade_lock SIGNAL parked WAIT_FOR go'; > --send alter table t1 rename t3 > > -connection default; > + connection default; eh? (here and many other changes in this file) > --echo connection: default > set debug_sync= 'now WAIT_FOR parked'; > > diff --git a/mysql-test/suite/gcol/inc/gcol_select.inc b/mysql-test/suite/gcol/inc/gcol_select.inc > index 2386c55fdbc..8f6314233bd 100644 > --- a/mysql-test/suite/gcol/inc/gcol_select.inc > +++ b/mysql-test/suite/gcol/inc/gcol_select.inc > @@ -872,7 +872,9 @@ CREATE TABLE t1(a INT); > INSERT INTO t1 VALUES(2147483647); > ALTER TABLE t1 ADD COLUMN h INT AS (a) VIRTUAL; > ALTER TABLE t1 CHANGE h i INT AS (a) VIRTUAL, ALGORITHM=COPY; > +--error ER_WARN_DATA_OUT_OF_RANGE,ER_ALTER_OPERATION_NOT_SUPPORTED_REASON why two? embedded, again? > ALTER TABLE t1 ADD COLUMN b SMALLINT AS (a) VIRTUAL, ALGORITHM=COPY, LOCK=NONE; > +--error ER_WARN_DATA_OUT_OF_RANGE,ER_ALTER_OPERATION_NOT_SUPPORTED_REASON > ALTER TABLE t1 ADD COLUMN e SMALLINT AS (a) VIRTUAL, ALGORITHM=COPY, LOCK=NONE; > ALTER TABLE t1 ADD COLUMN f SMALLINT AS (a) VIRTUAL, ALGORITHM=COPY, LOCK=SHARED; > ALTER TABLE t1 ADD COLUMN g SMALLINT AS (a) VIRTUAL, ALGORITHM=COPY, LOCK=EXCLUSIVE; > diff --git a/sql/handler.h b/sql/handler.h > index fe61666bf20..5e46fa59d7e 100644 > --- a/sql/handler.h > +++ b/sql/handler.h > @@ -3538,6 +3543,7 @@ class handler :public Sql_alloc > /** to be actually called to get 'check()' functionality*/ > int ha_check(THD *thd, HA_CHECK_OPT *check_opt); > int ha_repair(THD* thd, HA_CHECK_OPT* check_opt); > + virtual void open_read_view(){} why did you change to that from start_consistent_snapshot()? > void ha_start_bulk_insert(ha_rows rows, uint flags= 0) > { > DBUG_ENTER("handler::ha_start_bulk_insert"); > diff --git a/sql/log.cc b/sql/log.cc > index 70ceecc66f8..9beab80773c 100644 > --- a/sql/log.cc > +++ b/sql/log.cc > @@ -101,6 +102,9 @@ static int binlog_flush_cache(THD *thd, binlog_cache_mngr *cache_mngr, > Log_event *end_ev, bool all, bool using_stmt, > bool using_trx, bool is_ro_1pc); > > +int binlog_online_alter_commit(THD *thd, bool all); > +void binlog_online_alter_rollback(THD *thd, bool all); static? > + > static const LEX_CSTRING write_error_msg= > { STRING_WITH_LEN("error writing to the binary log") }; > > @@ -3565,6 +3613,91 @@ bool MYSQL_BIN_LOG::open_index_file(const char *index_file_name_arg, > } > > > +bool Event_log::open(const char *log_name, > + const char *new_name, ulong next_file_number, > + enum cache_type io_cache_type_arg) > +{ > + bool error= false; > + if (log_name || new_name) > + { > + error= MYSQL_LOG::open( > +#ifdef HAVE_PSI_INTERFACE > + 0, > +#endif > + log_name, LOG_NORMAL, new_name, next_file_number, io_cache_type_arg); > + } > + else > + { > +#ifdef HAVE_PSI_INTERFACE > + /* Keep the key for reopen */ > + m_log_file_key= 0; > +#endif > + error= init_io_cache(&log_file, -1, LOG_BIN_IO_SIZE, > + io_cache_type_arg, 0, 0, > + MYF(MY_WME | MY_NABP | MY_WAIT_IF_FULL)); > + > + log_state= LOG_OPENED; > + inited= true; > + } > + if (error) > + return error; > + > + longlong bytes_written= write_description_event( > + (enum_binlog_checksum_alg)binlog_checksum_options, > + encrypt_binlog, false); > + error= bytes_written < 0; > + return error; > +} > + > +longlong > +Event_log::write_description_event(enum_binlog_checksum_alg checksum_alg, > + bool encrypt, bool dont_set_created) > +{ > + Format_description_log_event s(BINLOG_VERSION); > + /* > + don't set LOG_EVENT_BINLOG_IN_USE_F for SEQ_READ_APPEND io_cache > + as we won't be able to reset it later > + */ > + if (io_cache_type == WRITE_CACHE) this was incorrectly rebased. in the previous patch you *moved* this block of code from MYSQL_BIN_LOG::open into a separate method. here you copied it. you forgot to delete it from MYSQL_BIN_LOG::open(). > + s.flags |= LOG_EVENT_BINLOG_IN_USE_F; > + s.checksum_alg= checksum_alg; > + > + crypto.scheme = 0; > + DBUG_ASSERT(s.checksum_alg != BINLOG_CHECKSUM_ALG_UNDEF); > + if (!s.is_valid()) > + return -1; > + s.dont_set_created= dont_set_created; > + if (write_event(&s, 0, &log_file)) > + return -1; > + > + if (encrypt) > + { > + uint key_version= encryption_key_get_latest_version(ENCRYPTION_KEY_SYSTEM_DATA); > + if (key_version == ENCRYPTION_KEY_VERSION_INVALID) > + { > + sql_print_error("Failed to enable encryption of binary logs"); > + return -1; > + } > + > + if (key_version != ENCRYPTION_KEY_NOT_ENCRYPTED) > + { > + if (my_random_bytes(crypto.nonce, sizeof(crypto.nonce))) > + return -1; > + > + Start_encryption_log_event sele(1, key_version, crypto.nonce); > + sele.checksum_alg= s.checksum_alg; > + if (write_event(&sele, 0, &log_file)) > + return -1; > + > + // Start_encryption_log_event is written, enable the encryption > + if (crypto.init(sele.crypto_scheme, key_version)) > + return -1; > + } > + } > + return (longlong)s.data_written; > +} > + > + > /** > Open a (new) binlog file. > > @@ -7252,6 +7477,160 @@ class CacheWriter: public Log_event_writer > bool first; > }; > > +int cache_copy(IO_CACHE *to, IO_CACHE *from) > +{ > + DBUG_ENTER("cache_copy"); > + if (reinit_io_cache(from, READ_CACHE, 0, 0, 0)) > + DBUG_RETURN(ER_ERROR_ON_WRITE); > + size_t bytes_in_cache= my_b_bytes_in_cache(from); > + > + do > + { > + my_b_write(to, from->read_pos, bytes_in_cache); > + > + from->read_pos += bytes_in_cache; > + bytes_in_cache= my_b_fill(from); > + if (from->error || to->error) > + DBUG_RETURN(ER_ERROR_ON_WRITE); > + } while (bytes_in_cache); > + > + DBUG_RETURN(0); > +} > + > +int binlog_online_alter_commit(THD *thd, bool all) > +{ > + DBUG_ENTER("online_alter_commit"); > + int error= 0; > +#ifdef HAVE_REPLICATION > + > + if (thd->online_alter_cache_list.empty()) > + DBUG_RETURN(0); > + > + bool is_ending_transaction= ending_trans(thd, all); > + > + for (auto &cache_mngr: thd->online_alter_cache_list) > + { > + auto *binlog= cache_mngr.share->online_alter_binlog; > + DBUG_ASSERT(binlog); > + > + error= binlog_flush_pending_rows_event(thd, > + /* > + do not set STMT_END for last event > + to leave table open in altering thd > + */ > + false, > + true, > + binlog, > + is_ending_transaction > + ? &cache_mngr.trx_cache > + : &cache_mngr.stmt_cache); > + if (is_ending_transaction) > + { > + mysql_mutex_lock(binlog->get_log_lock()); > + error= binlog->write_cache(thd, &cache_mngr.trx_cache.cache_log); > + > + mysql_mutex_unlock(binlog->get_log_lock()); > + } > + else > + { > + error= cache_copy(&cache_mngr.trx_cache.cache_log, > + &cache_mngr.stmt_cache.cache_log); Why do you need two caches here? It seems you could have just one trx_cache and if the statement fails you simply truncate it to the beginning of the statement. > + } > + > + if (error) > + { > + my_error(ER_ERROR_ON_WRITE, MYF(ME_ERROR_LOG), > + binlog->get_name(), errno); > + binlog_online_alter_cleanup(thd->online_alter_cache_list, > + is_ending_transaction); > + DBUG_RETURN(error); > + } > + } > + > + binlog_online_alter_cleanup(thd->online_alter_cache_list, > + is_ending_transaction); > + > + for (TABLE *table= thd->open_tables; table; table= table->next) > + { > + table->online_alter_cache= NULL; Why? > + } > +#endif // HAVE_REPLICATION > + DBUG_RETURN(error); > +} > + > +void binlog_online_alter_rollback(THD *thd, bool all) > +{ > +#ifdef HAVE_REPLICATION > + bool is_ending_trans= ending_trans(thd, all); > + > + /* > + This is a crucial moment that we are running through > + thd->online_alter_cache_list, and not through thd->open_tables to cleanup > + stmt cache, though both have it. The reason is that the tables can be closed > + to that moment in case of an error. > + The same reason applies to the fact we don't store cache_mngr in the table > + itself -- because it can happen to be not existing. > + Still in case if tables are left opened > + */ > + binlog_online_alter_cleanup(thd->online_alter_cache_list, is_ending_trans); > +#endif // HAVE_REPLICATION > +} > + > +SAVEPOINT** find_savepoint_in_list(THD *thd, LEX_CSTRING name, > + SAVEPOINT ** const list); > + > +SAVEPOINT* savepoint_add(THD *thd, LEX_CSTRING name, SAVEPOINT **list, > + int (*release_old)(THD*, SAVEPOINT*)); may be in transaction.h? > + > +int online_alter_savepoint_set(THD *thd, LEX_CSTRING name) > +{ > + > + DBUG_ENTER("binlog_online_alter_savepoint"); > +#ifdef HAVE_REPLICATION > + if (thd->online_alter_cache_list.empty()) > + DBUG_RETURN(0); > + > + if (savepoint_alloc_size < sizeof (SAVEPOINT) + sizeof(my_off_t)) > + savepoint_alloc_size= sizeof (SAVEPOINT) + sizeof(my_off_t); > + > + for (auto &cache: thd->online_alter_cache_list) > + { > + if (cache.share->db_type()->savepoint_set == NULL) > + continue; > + > + SAVEPOINT *sv= savepoint_add(thd, name, &cache.sv_list, NULL); wouldn't it be simpler to log SAVEPOINT and ROLLBACK/RELEASE as Query_log_event to the cache? Then you won't need to do any maintenance. > + if(unlikely(sv == NULL)) > + DBUG_RETURN(1); > + my_off_t *pos= (my_off_t*)(sv+1); > + *pos= cache.trx_cache.get_byte_position(); > + > + sv->prev= cache.sv_list; > + cache.sv_list= sv; > + } > +#endif > + DBUG_RETURN(0); > +} > + > +int online_alter_savepoint_rollback(THD *thd, LEX_CSTRING name) > +{ > + DBUG_ENTER("online_alter_savepoint_rollback"); > +#ifdef HAVE_REPLICATION > + for (auto &cache: thd->online_alter_cache_list) > + { > + if (cache.share->db_type()->savepoint_set == NULL) > + continue; > + > + SAVEPOINT **sv= find_savepoint_in_list(thd, name, &cache.sv_list); > + // sv is null if savepoint was set up before online table was modified > + my_off_t pos= *sv ? *(my_off_t*)(*sv+1) : 0; > + > + cache.trx_cache.restore_savepoint(pos); > + } > + > +#endif > + DBUG_RETURN(0); > +} > + > /* > Write the contents of a cache to the binary log. > > diff --git a/sql/rpl_rli.h b/sql/rpl_rli.h > index cc807852bf2..57757ecf3dd 100644 > --- a/sql/rpl_rli.h > +++ b/sql/rpl_rli.h > @@ -900,14 +900,20 @@ struct rpl_group_info > } > } > > - bool get_table_data(TABLE *table_arg, table_def **tabledef_var, TABLE **conv_table_var) const > + bool get_table_data(TABLE *table_arg, table_def **tabledef_var, > + TABLE **conv_table_var, > + const Copy_field *copy[], const Copy_field **copy_end) const > { > DBUG_ASSERT(tabledef_var && conv_table_var); > for (TABLE_LIST *ptr= tables_to_lock ; ptr != NULL ; ptr= ptr->next_global) > if (ptr->table == table_arg) > { > - *tabledef_var= &static_cast<RPL_TABLE_LIST*>(ptr)->m_tabledef; > - *conv_table_var= static_cast<RPL_TABLE_LIST*>(ptr)->m_conv_table; > + auto *rpl_table_list= static_cast<RPL_TABLE_LIST*>(ptr); > + if (rpl_table_list->m_tabledef_valid) when can it be false? > + *tabledef_var= &rpl_table_list->m_tabledef; > + *conv_table_var= rpl_table_list->m_conv_table; > + *copy= rpl_table_list->m_online_alter_copy_fields; > + *copy_end= rpl_table_list->m_online_alter_copy_fields_end; > DBUG_PRINT("debug", ("Fetching table data for table %s.%s:" > " tabledef: %p, conv_table: %p", > table_arg->s->db.str, table_arg->s->table_name.str, > diff --git a/sql/sql_table.cc b/sql/sql_table.cc > index b07efb29bba..9dfe2178667 100644 > --- a/sql/sql_table.cc > +++ b/sql/sql_table.cc > @@ -9533,6 +9539,19 @@ bool mysql_alter_table(THD *thd, const LEX_CSTRING *new_db, > has been already processed. > */ > table_list->required_type= TABLE_TYPE_NORMAL; > + > + > + if (alter_info->requested_lock == Alter_info::ALTER_TABLE_LOCK_SHARED > + || alter_info->requested_lock == Alter_info::ALTER_TABLE_LOCK_EXCLUSIVE > + || thd->locked_tables_mode == LTM_LOCK_TABLES > + || thd->lex->sql_command == SQLCOM_OPTIMIZE > + || alter_info->algorithm(thd) == Alter_info::ALTER_TABLE_ALGORITHM_NOCOPY) why only NOCOPY? What about INPLACE and INSTANT? the rest of the condition looks fine. > + online= false; > + > + if (online) > + { > + table_list->lock_type= TL_READ; > + } > > DEBUG_SYNC(thd, "alter_table_before_open_tables"); > > @@ -10961,6 +10983,58 @@ bool mysql_trans_commit_alter_copy_data(THD *thd) > DBUG_RETURN(error); > } > > +#ifdef HAVE_REPLICATION > +static int online_alter_read_from_binlog(THD *thd, rpl_group_info *rgi, > + Cache_flip_event_log *log) > +{ > + MEM_ROOT event_mem_root; > + Query_arena backup_arena; > + Query_arena event_arena(&event_mem_root, Query_arena::STMT_INITIALIZED); > + init_sql_alloc(key_memory_gdl, &event_mem_root, > + MEM_ROOT_BLOCK_SIZE, 0, MYF(0)); > + > + int error= 0; > + > + IO_CACHE *log_file= log->flip(); > + > + thd_progress_report(thd, 0, my_b_write_tell(log_file)); > + > + Abort_on_warning_instant_set old_abort_on_warning(thd, 0); > + do > + { > + const auto *descr_event= rgi->rli->relay_log.description_event_for_exec; > + auto *ev= Log_event::read_log_event(log_file, descr_event, false); > + if (!ev) > + break; > + > + ev->thd= thd; > + thd->set_n_backup_active_arena(&event_arena, &backup_arena); > + error= ev->apply_event(rgi); > + thd->restore_active_arena(&event_arena, &backup_arena); > + > + event_arena.free_items(); > + free_root(&event_mem_root, MYF(MY_KEEP_PREALLOC)); > + if (ev != rgi->rli->relay_log.description_event_for_exec) > + delete ev; > + thd_progress_report(thd, my_b_tell(log_file), thd->progress.max_counter); > + DEBUG_SYNC(thd, "alter_table_online_progress"); > + } while(!error); > + > + return error; > +} > +#endif // HAVE_REPLICATION > + > +static void online_alter_cleanup_binlog(THD *thd, TABLE_SHARE *s) > +{ > +#ifdef HAVE_REPLICATION > + if (!s->online_alter_binlog) > + return; > + // s->online_alter_binlog->reset_logs(thd, false, NULL, 0, 0); forgot to delete? > + s->online_alter_binlog->cleanup(); > + s->online_alter_binlog->~Cache_flip_event_log(); > + s->online_alter_binlog= NULL; > +#endif > +} > > static int > copy_data_between_tables(THD *thd, TABLE *from, TABLE *to, > @@ -11306,6 +11431,76 @@ copy_data_between_tables(THD *thd, TABLE *from, TABLE *to, > cleanup_done= 1; > to->file->extra(HA_EXTRA_NO_IGNORE_DUP_KEY); > > +#ifdef HAVE_REPLICATION > + if (likely(online && error < 0)) > + { > + Ha_trx_info *trx_info_save= thd->transaction->all.ha_list; > + thd->transaction->all.ha_list = NULL; why? > + thd_progress_next_stage(thd); > + Table_map_log_event table_event(thd, from, from->s->table_map_id, > + from->file->has_transactions()); > + Relay_log_info rli(false); > + rpl_group_info rgi(&rli); > + RPL_TABLE_LIST rpl_table(to, TL_WRITE, from, table_event.get_table_def(), > + copy, copy_end); > + Cache_flip_event_log *binlog= from->s->online_alter_binlog; > + rgi.thd= thd; > + rgi.tables_to_lock= &rpl_table; > + > + rgi.m_table_map.set_table(from->s->table_map_id, to); > + > + DBUG_ASSERT(binlog->is_open()); > + > + rli.relay_log.description_event_for_exec= > + new Format_description_log_event(4); > + > + // We restore bitmaps, because update event is going to mess up with them. > + to->default_column_bitmaps(); > + > + error= online_alter_read_from_binlog(thd, &rgi, binlog); > + > + DEBUG_SYNC(thd, "alter_table_online_before_lock"); > + > + int lock_error= > + thd->mdl_context.upgrade_shared_lock(from->mdl_ticket, MDL_EXCLUSIVE, > + (double)thd->variables.lock_wait_timeout); > + if (!error) > + error= lock_error; > + > + if (!error) > + { > + thd_progress_next_stage(thd); > + error= online_alter_read_from_binlog(thd, &rgi, binlog); > + } > + > + thd->transaction->all.ha_list = trx_info_save; > + } > + else if (unlikely(online)) // error was on copy stage > + { > + /* > + We need to issue a barrier to clean up gracefully. > + Without this, following possible: > + T1: ALTER TABLE starts > + T2: INSERT starts > + T1: ALTER TABLE fails with error (i.e. ER_DUP_KEY) > + T1: from->s->online_alter_binlog sets to NULL > + T2: INSERT committs > + T2: thd->online_alter_cache_list is not empty > + T2: binlog_commit: DBUG_ASSERT(binlog); is issued. 1. do you have a test for that? 2. can it be fixed, like, on iterating thd->online_alter_cache_list thd notices that from->s->online_alter_cache_list is NULL, so it simply discards the cache? Then you won't need MDL_SHARED_NO_WRITE > + */ > + // Ignore the return result. We already have an error. > + thd->mdl_context.upgrade_shared_lock(from->mdl_ticket, > + MDL_SHARED_NO_WRITE, > + thd->variables.lock_wait_timeout); > + } > +#endif > + > + if (error > 0 && !from->s->tmp_table) > + { > + /* We are going to drop the temporary table */ > + to->file->extra(HA_EXTRA_PREPARE_FOR_DROP); > + } > + > DEBUG_SYNC(thd, "copy_data_between_tables_before_reset_backup_lock"); > if (backup_reset_alter_copy_lock(thd)) > error= 1; Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] MDEV-16329 ALTER ONLINE TABLE
by Sergei Golubchik 21 Feb '22

21 Feb '22

Hi, Nikita, On Dec 13, Nikita Malyavin wrote: > > > -class binlog_cache_mngr { > > > +class binlog_cache_mngr: public ilist_node<> { > > > public: > > > binlog_cache_mngr(my_off_t param_max_binlog_stmt_cache_size, > > > my_off_t param_max_binlog_cache_size, > > > ulong *param_ptr_binlog_stmt_cache_use, > > > ulong *param_ptr_binlog_stmt_cache_disk_use, > > > ulong *param_ptr_binlog_cache_use, > > > - ulong *param_ptr_binlog_cache_disk_use) > > > + ulong *param_ptr_binlog_cache_disk_use, > > > + TABLE_SHARE *share) > > > - : last_commit_pos_offset(0), using_xa(FALSE), xa_xid(0) > > > + : last_commit_pos_offset(0), using_xa(FALSE), xa_xid(0), share(share) > > > > does it work, can one do it? I mean share(share) > > in other places, as far as I remember, arguments always use a distinct > > name, > > like share(share_arg) or share(share_) or something. > > > Yes, this is a language feature for constructor initializers. Okay. We don't use it though and it will stop almost everyone looking at this code, as it's all too easy to assume it's a mistake. So I'd recommend to rename the argument to avoid misunderstanding. Like, to `share_`. But also feel free to keep it your way, it's a valid C++ after all. > > > { > > > stmt_cache.set_binlog_cache_info(param_max_binlog_stmt_cache_size, > > > param_ptr_binlog_stmt_cache_use, > > > @@ -2116,9 +2144,58 @@ int binlog_commit(THD *thd, bool all, bool ro_1pc) ... > > > + else > > > + { > > > + error= cache_copy(&cache_mngr.trx_cache.cache_log, > > > + &cache_mngr.stmt_cache.cache_log); > > > > Why do you need two caches here? It seems you could have just one trx_cache > > and if the statement fails you simply truncate it to the beginning of > > the statement. > > > Didn't think about that, but seems so. Why would binlog need two caches > then? I don't really know. But, for example, I can think of a multi-table update that modifies both InnoDB and MyISAM tables and then fails after updating some rows. It'll be reverted in InnoDB, but not in MyISAM. With two caches InnoDB row events can go into stmt_cache and MyISAM row events can go into trx_cache. And later stmt_cache will be discarded, but MyISAM changes will persist. This multi-table case does not apply to online alter. Are there other cases when a separate stmt_cache might be needed? > > > + binlog_cache_mngr *const cache_mngr= thd->binlog_get_cache_mngr(); > > > + /* > > > + cache_mngr can be NULL in case if binlog logging is disabled. > > > + */ > > > if (!cache_mngr) > > > { > > > DBUG_ASSERT(WSREP(thd) || Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 592f57e25a5: MDEV-27760 event may non stop replicate in circular semisync setup
by Sergei Golubchik 17 Feb '22

17 Feb '22

Hi, Andrei, I'm sorry, but I don't understand that fix at all. Why did you move do_accept_own_server_id into Master_info if you only use it in one static function queue_event() ? Why did you change it under some complex condition that you moved somewhere up in the queue_event() when you could've done simply && opt_gtid_strict_mode ? On Feb 17, Andrei wrote: > revision-id: 592f57e25a5 (mariadb-10.6.5-82-g592f57e25a5) > parent(s): 8d742fe4acb > author: Andrei > committer: Andrei > timestamp: 2022-02-14 16:31:25 +0200 > message: > > MDEV-27760 event may non stop replicate in circular semisync setup > > diff --git a/sql/slave.cc b/sql/slave.cc > index 5ff40d8fb25..d5275d17460 100644 > --- a/sql/slave.cc > +++ b/sql/slave.cc > @@ -5037,6 +5037,7 @@ log space"); > mi->abort_slave= 0; > mi->slave_running= MYSQL_SLAVE_NOT_RUN; > mi->io_thd= 0; > + mi->do_accept_own_server_id= false; > /* > Note: the order of the two following calls (first broadcast, then unlock) > is important. Otherwise a killer_thread can execute between the calls and > @@ -6175,15 +6176,6 @@ static int queue_event(Master_info* mi, const uchar *buf, ulong event_len) > uchar new_buf_arr[4096]; > bool is_malloc = false; > bool is_rows_event= false; > - /* > - The flag has replicate_same_server_id semantics and is raised to accept > - a same-server-id event on the semisync slave, for both the gtid and legacy > - connection modes. > - Such events can appear as result of this server recovery so the event > - was created there and replicated elsewhere right before the crash. At recovery > - it could be evicted from the server's binlog. > - */ > - bool do_accept_own_server_id= false; > /* > FD_q must have been prepared for the first R_a event > inside get_master_version_and_clock() > @@ -6272,6 +6264,8 @@ static int queue_event(Master_info* mi, const uchar *buf, ulong event_len) > dbug_rows_event_count = 0; > };); > #endif > + s_id= uint4korr(buf + SERVER_ID_OFFSET); > + > mysql_mutex_lock(&mi->data_lock); > > switch (buf[EVENT_TYPE_OFFSET]) { > @@ -6713,6 +6707,20 @@ static int queue_event(Master_info* mi, const uchar *buf, ulong event_len) > > ++mi->events_queued_since_last_gtid; > inc_pos= event_len; > + > + if (s_id == global_system_variables.server_id && > + rpl_semi_sync_slave_enabled && > + mi->using_gtid != Master_info::USE_GTID_NO && > + opt_gtid_strict_mode) > + { > + rpl_gtid *state_gtid= mi->gtid_current_pos.find(event_gtid.domain_id); > + mi->do_accept_own_server_id= > + !state_gtid || state_gtid->seq_no < event_gtid.seq_no; > + } > + else > + { > + mi->do_accept_own_server_id= false; > + } > } > break; > /* > @@ -6900,7 +6908,6 @@ static int queue_event(Master_info* mi, const uchar *buf, ulong event_len) > */ > > mysql_mutex_lock(log_lock); > - s_id= uint4korr(buf + SERVER_ID_OFFSET); > /* > Write the event to the relay log, unless we reconnected in the middle > of an event group and now need to skip the initial part of the group that > @@ -6946,7 +6953,7 @@ static int queue_event(Master_info* mi, const uchar *buf, ulong event_len) > else > if ((s_id == global_system_variables.server_id && > !(mi->rli.replicate_same_server_id || > - (do_accept_own_server_id= rpl_semi_sync_slave_enabled))) || > + mi->do_accept_own_server_id)) || > event_that_should_be_ignored(buf) || > /* > the following conjunction deals with IGNORE_SERVER_IDS, if set > @@ -7006,7 +7013,7 @@ static int queue_event(Master_info* mi, const uchar *buf, ulong event_len) > } > else > { > - if (do_accept_own_server_id) > + if (mi->do_accept_own_server_id) > { > int2store(const_cast<uchar*>(buf + FLAGS_OFFSET), > uint2korr(buf + FLAGS_OFFSET) | LOG_EVENT_ACCEPT_OWN_F); > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] MDEV-10000 Add EXPLAIN FOR CONNECTION syntax support
by Sergey Petrunia 29 Jan '22

29 Jan '22

Hi Oleg, > commit 53d4d2852463714678b6e753ecffc1ecdde13891 > Author: Oleg Smirnov <olernov(a)gmail.com> > Date: Thu Jan 13 19:24:22 2022 +0300 > > MDEV-10000 Add support for FORMAT=JSON > > Makes possible calls to SHOW EXPLAIN and EXPLAIN FOR CONNECTION > providing output in JSON format: > - SHOW EXPLAIN FORMAT=JSON FOR $con > - EXPLAIN FORMAT=JSON FOR CONNECTION $con > The patch is mostly ok. I have a few cosmetic comments. Please follow the coding style and make sure the lines are no longer than 80 chars. > diff --git a/sql/sql_show.h b/sql/sql_show.h > index 3d7a4d1146c..8c8ae416699 100644 > --- a/sql/sql_show.h > +++ b/sql/sql_show.h > @@ -164,6 +164,9 @@ class Show_explain_request : public Apc_target::Apc_call > THD *target_thd; /* thd that we're running SHOW EXPLAIN for */ > THD *request_thd; /* thd that run SHOW EXPLAIN command */ > > + bool is_json_format= false; /* set to TRUE if you need the result in JSON > + format, FALSE - in traditional tabular */ > + Multi-line comments must precede the variable: /* Comment here... */ bool is_json_format; > /* If true, there was some error when producing EXPLAIN output. */ > bool failed_to_produce; > > diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy > index 74757970411..a7188762cbd 100644 > --- a/sql/sql_yacc.yy > +++ b/sql/sql_yacc.yy > @@ -13945,12 +13945,13 @@ show_param: > Lex->spname= $3; > Lex->sql_command = SQLCOM_SHOW_CREATE_EVENT; > } > - | describe_command FOR_SYM expr > + | describe_command opt_format_json FOR_SYM expr > { > Lex->sql_command= SQLCOM_SHOW_EXPLAIN; > - if (unlikely(prepare_schema_table(thd, Lex, 0, SCH_EXPLAIN))) > + if (unlikely(prepare_schema_table(thd, Lex, 0, > + Lex->explain_json ? SCH_EXPLAIN_JSON : SCH_EXPLAIN_TABULAR))) > MYSQL_YYABORT; > - add_value_to_list(thd, $3); > + add_value_to_list(thd, $4); > } > | IDENT_sys remember_tok_start wild_and_where > { > @@ -14123,12 +14124,18 @@ opt_describe_column: > ; Please fix identation of this rule to match of identation of others. Please also add a comment about the alternative syntax, both here and in the SHOW EXPLAIN rule. > explain_for_connection: > - describe_command FOR_SYM CONNECTION_SYM expr > + describe_command opt_format_json FOR_SYM CONNECTION_SYM expr > { > + Lex->wild=0; > + Lex->ident= null_clex_str; > + mysql_init_select(Lex); > + Lex->current_select->parsing_place= SELECT_LIST; > + Lex->create_info.init(); > Lex->sql_command= SQLCOM_SHOW_EXPLAIN; > - if (unlikely(prepare_schema_table(thd, Lex, 0, SCH_EXPLAIN))) > + if (unlikely(prepare_schema_table(thd, Lex, 0, > + Lex->explain_json ? SCH_EXPLAIN_JSON : SCH_EXPLAIN_TABULAR))) > MYSQL_YYABORT; > - add_value_to_list(thd, $4); > + add_value_to_list(thd, $5); > } > ; BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://petrunia.net

1 0

Re: [Maria-developers] MDEV-27021 Implement SHOW ANALYZE command
by Sergey Petrunia 28 Jan '22

28 Jan '22

Hi Oleg, > commit 92b85f455994ff9f18e30ad4b852109b3e3c5f04 > Author: Oleg Smirnov <olegs(a)teramind.co> > Date: Fri Dec 24 17:27:03 2021 +0300 > > MDEV-27021 Implement SHOW ANALYZE command > The patch looks good. A few cosmetic comments below. I've played with it and it looks usable, although I would expect that SHOW ANALYZE FORMAT=JSON would be the real killer. For SHOW ANALYZE FORMAT=JSON, I suspect we might need to add a few fields into it to make analysis easier (I'll elaborate later). Let's first get SHOW ANALYZE FORMAT=JSON to work without the extra columns. == Cosmetic comments == Please make sure there are no spaces at line ends. Some MariaDB developers feel strongly about not having them. (If your edior doesn't highlight them, "git citool" does). At risk of stating the obvious: * one should not remove the endspace if otherwise they are not changing the line of code. (such "dummy" changes make it hard to look at the revision history) * If one adds a line or changes an existing line, make sure the line doesnt have the end spaces. > diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt > index f31f21faeb0..e18f8c61067 100644 > --- a/sql/share/errmsg-utf8.txt > +++ b/sql/share/errmsg-utf8.txt > @@ -7224,7 +7224,7 @@ ER_NO_SUCH_TABLE_IN_ENGINE 42S02 > hindi "टेबल '%-.192s.%-.192s' इंजन में मौजूद नहीं है" > swe "Det finns ingen tabell som heter '%-.192s.%-.192s' i handlern" > ER_TARGET_NOT_EXPLAINABLE > - eng "Target is not running an EXPLAINable command" > + eng "Target is not running a command that allows to produce SHOW EXPLAIN or ANALYZE" . This will soon mean "SHOW EXPLAIN or ANALYZE or EXPLAIN FOR CONNECTION". Does "Target is not executing an operation with a query plan" sound better? > ER_CONNECTION_ALREADY_EXISTS > eng "Connection '%.*s' conflicts with existing connection '%.*s'" > ER_MASTER_LOG_PREFIX BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://petrunia.net

2 1

[Maria-developers] Igor please "review" : 768b5f70b59: Fix Item_func::is_simplifed_cond_processor
by Sergey Petrunia 27 Jan '22

27 Jan '22

Hello Igor, Could you please checkout the below patch? I've stumbled upon behavior of Item_func::is_simplifed_cond_processor which looks bad but I'm not sure if this way to fix it is correct. revision-id: 9149fcb8d504d7c1b027024df63c8120a65a7751 (mariadb-10.4.22-82-g9149fcb8d50) parent(s): b915f79e4e004fde4f6ac8f341afee980e11792b author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2022-01-27 22:58:11 +0300 message: Fix Item_func::is_simplifed_cond_processor The comment for that function says: TRUE if the function is knowingly TRUE or FALSE. I don't get why one would need to check for "!val_int()" in this case. --- mysql-test/main/having_cond_pushdown.result | 6 ------ sql/item_func.h | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/mysql-test/main/having_cond_pushdown.result b/mysql-test/main/having_cond_pushdown.result index 9b124296e3d..aade383a71f 100644 --- a/mysql-test/main/having_cond_pushdown.result +++ b/mysql-test/main/having_cond_pushdown.result @@ -3194,7 +3194,6 @@ EXPLAIN { "query_block": { "select_id": 1, - "const_condition": "1", "table": { "table_name": "t1", "access_type": "ALL", @@ -3536,7 +3535,6 @@ EXPLAIN { "query_block": { "select_id": 1, - "const_condition": "1 and 1", "table": { "table_name": "t1", "access_type": "ALL", @@ -3663,7 +3661,6 @@ EXPLAIN { "query_block": { "select_id": 1, - "const_condition": "1", "table": { "table_name": "t1", "access_type": "ALL", @@ -3791,7 +3788,6 @@ EXPLAIN { "query_block": { "select_id": 1, - "const_condition": "1", "filesort": { "sort_key": "t1.c", "temporary_table": { @@ -3859,7 +3855,6 @@ EXPLAIN { "query_block": { "select_id": 1, - "const_condition": "1 and 1", "table": { "table_name": "t1", "access_type": "ALL", @@ -3977,7 +3972,6 @@ EXPLAIN { "query_block": { "select_id": 1, - "const_condition": "1", "table": { "table_name": "t3", "access_type": "ALL", diff --git a/sql/item_func.h b/sql/item_func.h index b368e5872fe..2b02971d940 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -399,7 +399,7 @@ class Item_func :public Item_func_or_sum, With_sum_func_cache* get_with_sum_func_cache() { return this; } Item_func *get_item_func() { return this; } bool is_simplified_cond_processor(void *arg) - { return const_item() && !val_int(); } + { return const_item() /*&& !val_int();*/; } }; BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://petrunia.net

1 0

Re: [Maria-developers] MDEV-10000 Add EXPLAIN FOR CONNECTION syntax support
by Sergey Petrunia 21 Jan '22

21 Jan '22

Hi Oleg, > commit 8e6595cff0caa22d2bb81002505e52b5c52f5ab2 > Author: Oleg Smirnov <olernov(a)gmail.com> > Date: Tue Jan 11 20:25:11 2022 +0300 > > MDEV-10000 Add EXPLAIN FOR CONNECTION syntax support ... > diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy > index 442644eddd6..74757970411 100644 > --- a/sql/sql_yacc.yy > +++ b/sql/sql_yacc.yy > @@ -14121,6 +14122,15 @@ opt_describe_column: > } > ; > > +explain_for_connection: > + describe_command FOR_SYM CONNECTION_SYM expr > + { > + Lex->sql_command= SQLCOM_SHOW_EXPLAIN; > + if (unlikely(prepare_schema_table(thd, Lex, 0, SCH_EXPLAIN))) > + MYSQL_YYABORT; > + add_value_to_list(thd, $4); > + } > + ; > This fails to intialize the statement properly. For SHOW EXPLAIN, the following is done at statement start: show: SHOW { LEX *lex=Lex; lex->wild=0; lex->ident= null_clex_str; if (Lex->main_select_push()) MYSQL_YYABORT; mysql_init_select(lex); lex->current_select->parsing_place= SELECT_LIST; lex->create_info.init(); } If you don't do it, EXPLAIN FOR CONNECTION may produce crashes (e.g. on my machine it was crash in JOIN::prepare due to select_lex->inner_sum_func_list=0xa5a5a5a5a5). The rest of the code looks ok. Let's have another review iteration once the aboveis fixed. I've also found one difference with MySQL, which I am not sure if we should fix: In MariaDB, if the target is not running something that has an query plan, SHOW EXPLAIN (and now EXPLAIN FOR CONNECTION) will produce this error: MariaDB [test]> show explain for 5; ERROR 1933 (HY000): Target is not running an EXPLAINable command while in MySQL it can be one of two: 1. Target is idle: mysql> explain for connection 8; Query OK, 0 rows affected (0.00 sec) 2. Target is running a command that's not supported: mysql> explain for connection 8; ERROR 3012 (HY000): EXPLAIN FOR CONNECTION command is supported only for SELECT/UPDATE/INSERT/DELETE/REPLACE I'm not sure we should try to imitate this behavior. BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://petrunia.net

1 0

Re: [Maria-developers] 673ea509e2c: MDEV-26870 --skip-symbolic-links does not disallow .isl file creation
by Sergei Golubchik 21 Jan '22

21 Jan '22

Hi, Marko, On Jan 20, Marko Mäkelä wrote: > revision-id: 673ea509e2c (mariadb-10.2.40-230-g673ea509e2c) > parent(s): d28d3aee108 > author: Marko Mäkelä > committer: Marko Mäkelä > timestamp: 2022-01-20 16:13:08 +0200 > message: > > MDEV-26870 --skip-symbolic-links does not disallow .isl file creation > > The InnoDB DATA DIRECTORY attribute is not implemented via > symbolic links but something similar, *.isl files that contain > the names of data files. > > InnoDB failed to reject the DATA DIRECTORY attribute even though > the server was started with --skip-symbolic-links. > > Because TRUNCATE TABLE cannot easily return an error, it will keep > rebuilding tables even if they carry the DATA DIRECTORY attribute. > > diff --git a/mysql-test/suite/encryption/t/innodb-first-page-read.test b/mysql-test/suite/encryption/t/innodb-first-page-read.test > index d661e4565d2..a0b3ca3f0ff 100644 > --- a/mysql-test/suite/encryption/t/innodb-first-page-read.test > +++ b/mysql-test/suite/encryption/t/innodb-first-page-read.test > @@ -1,6 +1,7 @@ > -- source include/have_innodb.inc > -- source include/have_file_key_management_plugin.inc > -- source include/not_embedded.inc > +-- source include/have_symlink.inc when can have_symlink be false? Why InnoDB tests depend on server's symlink support? InnoDB does not use symlinks, it uses isl files. > --disable_warnings > SET GLOBAL innodb_file_format = `Barracuda`; > diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc > index 5d78e64a06b..a8b16ba1eb0 100644 > --- a/storage/innobase/handler/ha_innodb.cc > +++ b/storage/innobase/handler/ha_innodb.cc > @@ -11710,9 +11710,15 @@ create_table_info_t::create_options_are_invalid() > break; > } > > - if (m_create_info->data_file_name > - && m_create_info->data_file_name[0] != '\0' > - && !create_option_data_directory_is_valid()) { > + if (!m_create_info->data_file_name > + || !m_create_info->data_file_name[0]) { > + } else if (!my_use_symdir) { > + push_warning( > + m_thd, Sql_condition::WARN_LEVEL_WARN, > + ER_ILLEGAL_HA_CREATE_OPTION, > + "InnoDB: DATA DIRECTORY requires HAVE_SYMLINK."); > + ret = "DATA DIRECTORY"; MyISAM does push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, WARN_OPTION_IGNORED, ER_THD(thd, WARN_OPTION_IGNORED), "DATA DIRECTORY"); Archive does if (create_info->data_file_name) my_error(WARN_OPTION_IGNORED, MYF(ME_WARNING), "DATA DIRECTORY"); I suggest you do one of the above too. > + } else if (!create_option_data_directory_is_valid()) { > ret = "DATA DIRECTORY"; > } > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 3

Re: [Maria-developers] 99e2a49acfc: MDEV-27018 IF and COALESCE lose "json" property
by Sergei Golubchik 20 Jan '22

20 Jan '22

Hi, Alexander! On Jan 13, Alexander Barkov wrote: > revision-id: 99e2a49acfc (mariadb-10.5.13-33-g99e2a49acfc) > parent(s): 2776635cb98 > author: Alexander Barkov > committer: Alexander Barkov > timestamp: 2022-01-10 18:05:55 +0400 > message: > > MDEV-27018 IF and COALESCE lose "json" property > > Hybrid functions (IF, COALESCE, etc) did not preserve the JSON property > from their arguments. The same problem was repeatable for single row subselects. > > The problem happened because the method Item::is_json_type() was inconsistently > implemented across the Item hierarchy. For example, Item_hybrid_func > and Item_singlerow_subselect did not override is_json_type(). > > Solution: > > - Removing Item::is_json_type() > > - Implementing specific JSON type handlers: > Type_handler_string_json > Type_handler_varchar_json > Type_handler_tiny_blob_json > Type_handler_blob_json > Type_handler_medium_blob_json > Type_handler_long_blob_json > > - Reusing the existing data type infrastructure to pass JSON > type handlers across all item types, including classes Item_hybrid_func > and Item_singlerow_subselect. Note, these two classes themselves do not > need any changes! > > - Extending the data type infrastructure so data types can inherit > their properties (e.g. aggregation rules) from their base data types. > E.g. VARCHAR/JSON acts as VARCHAR, LONGTEXT/JSON acts as LONGTEXT > when mixed to a non-JSON data type. This is done by: > - adding virtual method Type_handler::type_handler_base() > - adding class Recursive_type_pair_iterator > - refactoring Type_handler_hybrid_field_type methods > aggregate_for_result(), aggregate_for_min_max(), > aggregate_for_num_op() to use Recursive_type_pair_iterator. > > This change also fixes: > > MDEV-27361 Hybrid functions with JSON arguments do not send format metadata > > diff --git a/sql/field.cc b/sql/field.cc > index 2c768527ced..8fa3bbd538c 100644 > --- a/sql/field.cc > +++ b/sql/field.cc > @@ -7277,6 +7277,19 @@ bool Field_longstr::send(Protocol *protocol) > } > > > +const Type_handler *Field_string::type_handler() const > +{ > + if (is_var_string()) > + return &type_handler_var_string; shouldn't it be after json check? > + /* > + This is a temporary solution and will be fixed soon (in 10.9?). > + Type_handler_string_json will provide its own Field_string_json. > + */ > + if (Type_handler_json_common::has_json_valid_constraint(this)) > + return &type_handler_string_json; > + return &type_handler_string; > +} > + > /* Copy a string and fill with space */ > > int Field_string::store(const char *from, size_t length,CHARSET_INFO *cs) > diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc > index ddf5fc32ea4..81003be4656 100644 > --- a/sql/item_jsonfunc.cc > +++ b/sql/item_jsonfunc.cc > @@ -1441,6 +1441,32 @@ longlong Item_func_json_contains_path::val_int() > } > > > +/* > + This reproduces behavior according to the former > + Item_func_conv_charset::is_json_type() which returned args[0]->is_json_type(). > + JSON functions with multiple string input with different character sets > + wrap some arguments into Item_func_conv_charset. So the former > + Item_func_conv_charset::is_json_type() took the JSON propery from args[0], > + i.e. from the original argument before the conversion. > + This is probably not always correct because an *explicit* > + `CONVERT(arg USING charset)` is actually a general purpose string > + expression, not a JSON expression. > +*/ > +static bool is_json_type(const Item *item) > +{ > + for ( ; ; ) > + { > + if (Type_handler_json_common::is_json_type_handler(item->type_handler())) > + return true; > + const Item_func_conv_charset *func; > + if (!(func= dynamic_cast<const Item_func_conv_charset*>(item))) > + return false; > + item= func->arguments()[0]; can you have nested CONVERT()'s ? > + } > + return false; > +} > + > + > static int append_json_value(String *str, Item *item, String *tmp_val) > { > if (item->type_handler()->is_bool_type()) > diff --git a/sql/sql_type_json.h b/sql/sql_type_json.h > index 6c4ee8cb2eb..4a394809a06 100644 > --- a/sql/sql_type_json.h > +++ b/sql/sql_type_json.h > @@ -21,18 +21,145 @@ ... > +template <class BASE, const Named_type_handler<BASE> &thbase> > +class Type_handler_general_purpose_string_to_json: > + public BASE, > + public Type_handler_json_common > { ... > + bool Item_hybrid_func_fix_attributes(THD *thd, > + const char *name, > + Type_handler_hybrid_field_type *hybrid, > + Type_all_attributes *attr, > + Item **items, uint nitems) > + const override > + { > + if (BASE::Item_hybrid_func_fix_attributes(thd, name, hybrid, attr, > + items, nitems)) > + return true; > + /* > + The above call can change the type handler on "hybrid", e.g. > + choose a proper BLOB type handler according to the calculated max_length. > + Convert general purpose string type handler to its JSON counterpart. > + This makes hybrid functions preserve JSON data types, e.g.: > + COALESCE(json_expr1, json_expr2) -> JSON > + */ > + hybrid->set_handler(json_type_handler_from_generic(hybrid->type_handler())); When this line would change hybrid->type_handler() ? > + return false; > + } > }; ..... > diff --git a/sql/sql_type.cc b/sql/sql_type.cc > index c1801c1ae3e..3b2753d80a6 100644 > --- a/sql/sql_type.cc > +++ b/sql/sql_type.cc > @@ -1755,19 +1755,110 @@ const Type_handler *Type_handler_typelib::cast_to_int_type_handler() const > > /***************************************************************************/ > > +class Recursive_type_pair_iterator > +{ > + const Type_handler *m_a; > + const Type_handler *m_b; > + uint m_switched_to_base_count; > +public: > + Recursive_type_pair_iterator(const Type_handler *a, > + const Type_handler *b, > + uint switched_to_base_count= 0) > + :m_a(a), m_b(b), m_switched_to_base_count(switched_to_base_count) > + { } > + const Type_handler *a() const { return m_a; } > + const Type_handler *b() const { return m_b; } > + Recursive_type_pair_iterator base() const > + { > + Recursive_type_pair_iterator res(m_a->type_handler_base(), > + m_b->type_handler_base()); > + res.m_switched_to_base_count= (res.m_a != NULL) + (res.m_b != NULL); > + if (res.m_a == NULL) > + res.m_a= m_a; > + if (res.m_b == NULL) > + res.m_b= m_b; > + return res; that's an unusual semantics, not what I would expect from an iterator. It'd expect it to iterare with it++ or it.next() but not it = it.base() > + } > + bool done() const > + { > + switch (m_switched_to_base_count) > + { > + case 2: I don't think case 2 is possible anymore. ... > + } > + } > +}; > + > + > bool > Type_handler_hybrid_field_type::aggregate_for_result(const Type_handler *other) > { > - const Type_handler *hres; > - const Type_collection *c; > - if (!(c= Type_handler::type_collection_for_aggregation(m_type_handler, other)) || > - !(hres= c->aggregate_for_result(m_type_handler, other))) > - hres= type_handler_data-> > - m_type_aggregator_for_result.find_handler(m_type_handler, other); > - if (!hres) > - return true; > - m_type_handler= hres; > - return false; > + Recursive_type_pair_iterator it(m_type_handler, other); > + for ( ; ; ) do you really still need to do recursive? in what cases? > + { > + const Type_handler *hres; > + const Type_collection *c; > + if (((c= Type_handler::type_collection_for_aggregation(it.a(), it.b())) && > + (hres= c->aggregate_for_result(it.a(), it.b()))) || > + (hres= type_handler_data-> > + m_type_aggregator_for_result.find_handler(it.a(), it.b()))) > + { > + m_type_handler= hres; > + return false; > + } > + if ((it= it.base()).done()) > + break; > + } > + return true; > } Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 4

Re: [Maria-developers] e4ea1544097: MDEV-17124: mariadb 10.1.34, views and prepared statements: ERROR 1615 (HY000): Prepared statement needs to be re-prepared
by Sergei Golubchik 20 Jan '22

20 Jan '22

Hi, Oleksandr, On Jan 19, Oleksandr Byelkin wrote: > revision-id: e4ea1544097 (mariadb-10.2.40-198-ge4ea1544097) > parent(s): fd7a553cb80 > author: Oleksandr Byelkin > committer: Oleksandr Byelkin > timestamp: 2022-01-19 11:12:57 +0100 > message: > > MDEV-17124: mariadb 10.1.34, views and prepared statements: ERROR 1615 (HY000): Prepared statement needs to be re-prepared > > The problem is that if table definition cache (TDC) is full of real tables > which are in tables cache, view definition can not stay there so will be > removed by its own underlying tables. > In situation above old mechanism of detection matching definition in PS > and current version always require reprepare and so prevent executing > the PS. > > One work around is to increase TDC, other - improve version check for > views/triggers (which is done here). Now in suspicious cases we check: > - MD5 of the view to be sure that version really have changed; > - time of creation of a trigger related to time of statement preparation. > > diff --git a/mysql-test/r/ps_ddl.result b/mysql-test/r/ps_ddl.result > index e69c6e06193..005c78a0319 100644 > --- a/mysql-test/r/ps_ddl.result > +++ b/mysql-test/r/ps_ddl.result > @@ -214,7 +214,7 @@ new trigger: 10 > drop trigger t1_bd; > set @val=11; > execute stmt using @val; > -call p_verify_reprepare_count(1); > +call p_verify_reprepare_count(0); > SUCCESS > > select @message; > @@ -831,8 +831,8 @@ a b c > 20 40 100 > 30 60 150 > call p_verify_reprepare_count(1); > -SUCCESS > - > +ERROR > +Expected: 1, actual: 0 > # Check that we properly handle ALTER VIEW statements. > execute stmt; > a b c > @@ -882,9 +882,9 @@ a b c > 10 50 60 > 20 100 120 > 30 150 180 > -call p_verify_reprepare_count(1); > -SUCCESS > - > +call p_verify_reprepare_count(0); > +ERROR > +Expected: 0, actual: 1 > execute stmt; > a b c you forgot to update the expected number or reprepares in two places. > diff --git a/sql/parse_file.cc b/sql/parse_file.cc > index 999f53bd681..3c750f4df29 100644 > --- a/sql/parse_file.cc > +++ b/sql/parse_file.cc > @@ -145,6 +145,7 @@ write_parameter(IO_CACHE *file, uchar* base, File_option *parameter) > > switch (parameter->type) { > case FILE_OPTIONS_STRING: > + case FILE_OPTIONS_FIXSTRING: > { > LEX_STRING *val_s= (LEX_STRING *)(base + parameter->offset); > if (my_b_write(file, (const uchar *)val_s->str, val_s->length)) > @@ -830,6 +831,22 @@ File_parser::parse(uchar* base, MEM_ROOT *mem_root, > } > ptr= eol+1; > break; > + case FILE_OPTIONS_FIXSTRING: > + { > + /* string have to be allocated already and length set */ > + LEX_STRING *val= (LEX_STRING *)(base + parameter->offset); > + DBUG_ASSERT(val->length != 0); > + if (ptr[val->length] != '\n') > + { > + my_error(ER_FPARSER_ERROR_IN_PARAMETER, MYF(0), > + parameter->name.str, line); > + DBUG_RETURN(TRUE); > + } > + memcpy(val->str, ptr, val->length); > + val->str[val->length]= '\0'; > + ptr+= (val->length + 1); > + break; I'd suggest to use FILE_OPTIONS_STRING here, it's much simpler and it's known to work. This doesn't check that end - ptr < val->length, and it doesn't check that even if ptr[val->length] == '\n', then it's the first '\n' after ptr. It's all possible to fix, but using FILE_OPTIONS_STRING is just easier. > + } > case FILE_OPTIONS_TIMESTAMP: > { > /* string have to be allocated already */ > diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc > index bb1a99d9eef..b9198647f98 100644 > --- a/sql/sql_prepare.cc > +++ b/sql/sql_prepare.cc > @@ -4177,6 +4177,10 @@ bool Prepared_statement::prepare(const char *packet, uint packet_len) > Query_arena *old_stmt_arena; > DBUG_ENTER("Prepared_statement::prepare"); > DBUG_ASSERT(m_sql_mode == thd->variables.sql_mode); > + > + // The same format as for triggers to compare > + create_time= make_trigger_time(thd->query_start(), > + thd->query_start_sec_part()); Better use hrtime here, you might compare it with more than triggers in the future. For triggers, you can multiply it by 10000 for comparison. Or you can start storing trigger time with a microsecond precision - that's even better. > /* > If this is an SQLCOM_PREPARE, we also increase Com_prepare_sql. > However, it seems handy if com_stmt_prepare is increased always, > diff --git a/sql/sql_trigger.cc b/sql/sql_trigger.cc > index 52817ef65ae..38febce4fef 100644 > --- a/sql/sql_trigger.cc > +++ b/sql/sql_trigger.cc > @@ -929,7 +933,21 @@ bool Table_triggers_list::create_trigger(THD *thd, TABLE_LIST *tables, > > if (!sql_create_definition_file(NULL, &file, &triggers_file_type, > (uchar*)this, triggers_file_parameters)) > + { > + // delay if needed to be sure that time of creation is unique > + for (;;) > + { > + my_hrtime_t hrtime= my_hrtime(); > + ulonglong current_time= make_trigger_time(hrtime_to_my_time(hrtime), > + hrtime_sec_part(hrtime)); > + if (trigger->create_time == current_time) > + pthread_yield(); > + else > + break; > + } This shouldn't be needed. Better 1. store statement->create_time with microsecond precision 2. store trigger->create_time with microsecond precision 3. at the end of prepare wait (like above) to make sure statement->create_time is larger than all triggers' create_time. this will be a no-op unless someone created a trigger and prepared in the same microsecond. Safety check will ensure you won't wait longer than one microsecond. > DBUG_RETURN(false); > + } > > err_with_cleanup: > /* Delete .TRN file */ > diff --git a/sql/sql_view.cc b/sql/sql_view.cc > index 020830483c5..830a42b3cf3 100644 > --- a/sql/sql_view.cc > +++ b/sql/sql_view.cc > @@ -1131,7 +1141,31 @@ static int mysql_register_view(THD *thd, TABLE_LIST *view, > DBUG_RETURN(error); > } > > +/** > + Check is TABLE_LEST and SHARE match > + @param[in] view TABLE_LIST of the view > + @param[in] share Share object of view > + > + @return false on error or misspatch > +*/ > > +bool mariadb_view_version_get(TABLE_SHARE *share) > +{ > + DBUG_ASSERT(share->is_view); > + > + if (!(share->tabledef_version.str= > + (uchar*) alloc_root(&share->mem_root, VIEW_MD5_LEN + 1))) > + return TRUE; > + share->tabledef_version.length= VIEW_MD5_LEN; > + > + DBUG_ASSERT(share->view_def != NULL); > + if (share->view_def->parse((uchar *) &share->tabledef_version, NULL, > + view_md5_parameters, 1, > + &file_parser_dummy_hook)) > + return TRUE; > + DBUG_ASSERT(share->tabledef_version.length == VIEW_MD5_LEN); > + return FALSE; > +} I don't like using md5 as a uuid. md5 collisions are very much possible. Better options would be 1. use a timestamp, but store it with microsecond precisions, same logic as with triggers. 2. use a true tabledef uuid, new field in the view frm, same logic as with tables. timestamp is better, because it already exists in frms, and any old view will naturally have a timestamp before now(), so it doesn't matter that it won't have microseconds. uuid is better, because it's more universal, can be used for more than just reprepare checks. But for now I think timestamp benefits outweigh that. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 8f8e7ad5c80: MDEV-27217 DELETE partition selection doesn't work for history partitions
by Sergei Golubchik 13 Jan '22

13 Jan '22

Hi, Aleksey! ok to push. see one comment below. On Jan 13, Aleksey Midenkov wrote: > revision-id: 8f8e7ad5c80 (mariadb-10.3.31-80-g8f8e7ad5c80) > parent(s): 88f8aa20bba > author: Aleksey Midenkov > committer: Aleksey Midenkov > timestamp: 2022-01-12 21:47:03 +0300 > message: > > MDEV-27217 DELETE partition selection doesn't work for history partitions > > LIMIT history switching requires the number of history partitions to > be marked for read: from first to last non-empty plus one empty. The > least we can do is to fail with error message if the needed partition > was not marked for read. As this is handler interface we require new > handler error code to display user-friendly error message. > > diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc > index bd73642cd0d..2f945f29ed1 100644 > --- a/sql/ha_partition.cc > +++ b/sql/ha_partition.cc > @@ -9872,6 +9874,11 @@ void ha_partition::print_error(int error, myf errflag) > m_part_info->print_no_partition_found(table, errflag); > DBUG_VOID_RETURN; > } > + else if (error == HA_ERR_PARTITION_LIST) > + { > + handler::print_error(error, errflag); > + DBUG_VOID_RETURN; > + } this seems to be redundant, the else branch at the end covers it. > else if (error == HA_ERR_ROW_IN_WRONG_PARTITION) > { Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] ef70e809a40: MDEV-27217 DELETE partition selection doesn't work for history partitions
by Sergei Golubchik 12 Jan '22

12 Jan '22

Hi, Aleksey! On Jan 12, Aleksey Midenkov wrote: > revision-id: ef70e809a40 (mariadb-10.3.31-80-gef70e809a40) > parent(s): 88f8aa20bba > author: Aleksey Midenkov > committer: Aleksey Midenkov > timestamp: 2021-12-23 04:15:39 +0300 > message: > > MDEV-27217 DELETE partition selection doesn't work for history partitions > > LIMIT history switching requires the number of history partitions to > be marked for read: from first to last non-empty plus one empty. The > least we can do is to fail with error message if the needed partition > was not marked for read. As this is handler interface we require new > handler error code to display user-friendly error message. > > Switching by INTERVAL works out-of-the-box with > ER_ROW_DOES_NOT_MATCH_GIVEN_PARTITION_SET error. Code-wise it's pretty much fine. Two problems related to error messages: We cannot have part of the error message a hard-coded English phrase. It doesn't work with localized error messages: "Невозможно выбрать раздел: modifying system versioned table" Also we cannot add new error messages in 10.3. Or any old GA. Currently you can add new error messages to 10.6 or any later version. You can reuse ER_UNUSED_1 for your new error message, that'll work. And, because you cannot have English phrases in it, I'd suggest something more generic, like Not allowed for system-versioned table %`s.%`s In the context of delete from t1 partition (p1); ERROR HY000: Not allowed for system-versioned table `test`.`t1` it seems to be quite clear. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 1

Re: [Maria-developers] 7a746072a5e: MDEV-26878: Query failing with syntax error sets ROW_NUMBER to non-zero
by Sergei Golubchik 12 Jan '22

12 Jan '22

Hi, Rucha! On Jan 12, Rucha Deodhar wrote: > On Tue, Jan 11, 2022 at 11:49 PM Sergei Golubchik <serg(a)mariadb.org> wrote: > > > +--error ER_PARSE_ERROR > > > +INSERT INTO t1 VALUES XXX (1),(2); > > > > > diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy > > > index 8ebefbb3d82..47ed3ded4a7 100644 > > > --- a/sql/sql_yacc.yy > > > +++ b/sql/sql_yacc.yy > > > @@ -8426,6 +8426,8 @@ query_specification_start: > > > { > > > SELECT_LEX *sel; > > > LEX *lex= Lex; > > > + if (lex->sql_command == SQLCOM_INSERT) > > > + thd->get_stmt_da()->inc_current_row_for_warning(); > > > if (!(sel= lex->alloc_select(TRUE)) || lex->push_select(sel)) > > > MYSQL_YYABORT; > > > > better not to have an if() in this rule that's used almost for > > everything. > > When we have INSERT...SELECT counter has to be 1 for errors like > ER_ILLEGAL_VALUE_FOR_TYPE which are caught during parsing. > Example: > INSERT INTO t1 SELECT a, b, 1.79769313486232e+308 FROM t2; > > So I added if() to increment the counter if we are parsing INSERT so > that ROW_NUMBER for statements like the one above gives ROW_NUMBER=1. I see. But you put a condition on every query's execution path to fix the case that, I believe, literally nobody cares about. I don't think any user ever will check the ROW_NUMBER after the syntax error :) And if someone will, we can say that in INSERT INTO t1 VALUES XXX (1),(2); the first row starts after VALUES, so ROW_NUMBER=1 is expected. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 9ea85a70a75: MDEV-24654 GTID event falsely marked transactional
by Sergei Golubchik 11 Jan '22

11 Jan '22

Hi, Andrei! On Jan 11, Andrei Elkin wrote: > Howdy, Sergei! > > To the question of the usage of trX cache by non-trX let me answer > broadly to mention @@binlog_direct_non_transactional_update = false > leads to aggregation of mixed, say innodb + myisam, events in trx > cache. That's different. The bug summary is "GTID event falsely marked transactional", meaning, the group of events is not transactional and it's falsely marked as transactional. If you have an innodb/myisam mix, it is correctly marked transactional. You need to have only non-transactional events in the transactional cache. And I'm trying to understand how can that happen. > I may need to process deeper the referred comments though. > > Next to the commit's idea, it actually covers your concern. I wonder > on the 'Wrong' conclusion. There is no intent to xidify anything > extra. Th commit assumes xid eVent is created, before Gtid one, > accordingly. And that fact is propagated to Gtid ctor. Your commit, if I'm not mistaken, assumes that "last event is Xid if and only if the cache contains a transaction". Which is incorrect, a transaction does not necesarily have to end with a Xid event. /Sergei > On Mon, 10 Jan 2022, 20:56 Sergei Golubchik, <serg(a)mariadb.org> wrote: > > > Hi, Andrei! > > > > On Jan 10, Andrei Elkin wrote: > > > revision-id: 9ea85a70a75 (mariadb-10.2.40-4-g9ea85a70a75) > > > parent(s): 160d97a4aaa > > > author: Andrei Elkin > > > committer: Andrei Elkin > > > timestamp: 2021-08-08 14:20:57 +0300 > > > message: > > > > > > MDEV-24654 GTID event falsely marked transactional > > > > > > GTID event can be falsely marked transactional in few cases including > > > binary-logging on the slave side upon execution. > > > In some execution branches bin-logging of non-transactional > > > events can be done through transactional cache. E.g see comments > > > in THD::binlog_write_table_map(). > > > > I saw that and still couldn't understand why bin-logging of > > non-transactional events can be done thr-ough transactional cache. > > > > > In order to not create the false 'trans' tag the fact of Xid event > > > logging is checked to compute correct argument to Gtid_log_event > > > ctor. > > > > This is wrong. Transactions only end with a Xid event if all > > participating engines support XA. Try to create an InnoDB+FederatedX > > transaction (I just did) and it'll end with Query_log_event("COMMIT") > > > > > diff --git a/sql/log.cc b/sql/log.cc > > > index 1d9b4645421..7fef2e0d739 100644 > > > --- a/sql/log.cc > > > +++ b/sql/log.cc > > > @@ -8118,9 +8118,11 @@ > > MYSQL_BIN_LOG::write_transaction_or_stmt(group_commit_entry *entry, > > > uint64 commit_id) > > > { > > > binlog_cache_mngr *mngr= entry->cache_mngr; > > > + bool has_xid= entry->end_event->get_type_code() == XID_EVENT; > > > DBUG_ENTER("MYSQL_BIN_LOG::write_transaction_or_stmt"); > > > > > > - if (write_gtid_event(entry->thd, false, entry->using_trx_cache, > > commit_id)) > > > + if (write_gtid_event(entry->thd, false, > > > + entry->using_trx_cache && has_xid, commit_id)) > > > DBUG_RETURN(ER_ERROR_ON_WRITE); > > > > > > if (entry->using_stmt_cache && !mngr->stmt_cache.empty() && > > > > > Regards, > > Sergei > > VP of MariaDB Server Engineering > > and security(a)mariadb.org > > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 2

Re: [Maria-developers] 9ea85a70a75: MDEV-24654 GTID event falsely marked transactional
by Sergei Golubchik 11 Jan '22

11 Jan '22

Hi, Andrei! On Jan 11, Andrei Elkin wrote: > On Tue, Jan 11, 2022 at 1:06 PM Sergei Golubchik <serg(a)mariadb.org> wrote: > > > Hi, Andrei! > > > > On Jan 11, Andrei Elkin wrote: > > > Howdy, Sergei! > > > > > > To the question of the usage of trX cache by non-trX let me answer > > > broadly to mention @@binlog_direct_non_transactional_update = false > > > leads to aggregation of mixed, say innodb + myisam, events in trx > > > cache. > > > > That's different. The bug summary is "GTID event falsely marked > > transactional", meaning, the group of events is not transactional and > > it's falsely marked as transactional. If you have an innodb/myisam > > mix, it is correctly marked transactional. > > > > You need to have only non-transactional events in the transactional > > cache. And I'm trying to understand how can that happen. > > Got your very specific point is about a pure myisam can be trx-cached. > This is OPTION_BINLOG_GTID > commit 5426facdcbfba2d78dab3c709cbf278073383b7c 's decision > >> - All tables are regarded as transactional tables in the binary log > (to ensure things are executed exactly as on the master) Okay, thanks, that answers my question Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 7a746072a5e: MDEV-26878: Query failing with syntax error sets ROW_NUMBER to non-zero
by Sergei Golubchik 11 Jan '22

11 Jan '22

Hi, Rucha! On Jan 11, Rucha Deodhar wrote: > revision-id: 7a746072a5e (mariadb-10.7.1-26-g7a746072a5e) > parent(s): 7dfaded9625 > author: Rucha Deodhar > committer: Rucha Deodhar > timestamp: 2022-01-05 12:05:22 +0530 > message: > > MDEV-26878: Query failing with syntax error sets ROW_NUMBER to non-zero > > Analysis: m_current_row_for_warning counter is set to 0 at the beginning > of parsing. > Fix: Reset it to 1 and increment it before starting to parse insert values, > during SET and SELECT for INSERT...SET statement and INSERT...SELECT > respectively. > > diff --git a/mysql-test/main/get_diagnostics.result b/mysql-test/main/get_diagnostics.result > diff --git a/mysql-test/main/get_diagnostics.test b/mysql-test/main/get_diagnostics.test > index e8d81dca1e6..467f624c49c 100644 > --- a/mysql-test/main/get_diagnostics.test > +++ b/mysql-test/main/get_diagnostics.test > @@ -1687,3 +1687,18 @@ GET DIAGNOSTICS CONDITION 3 @n = ROW_NUMBER; > SELECT @n; > > DROP TABLE t; > + > +--echo # > +--echo # MDEV-26878: Query failing with syntax error sets ROW_NUMBER to > +--echo # non-zero > +--echo # > + > +CREATE TABLE t1 (id1 INT); > + > +--error ER_PARSE_ERROR > +INSERT INTO t1 VALUES XXX (1),(2); please add a test, like, INTO t1 VALUES (1), (ALL); what ROW_NUMBER will it show? > + > +GET DIAGNOSTICS CONDITION 1 @n= ROW_NUMBER; > +SELECT @n; > + > +DROP TABLE t1; > diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy > index 8ebefbb3d82..47ed3ded4a7 100644 > --- a/sql/sql_yacc.yy > +++ b/sql/sql_yacc.yy > @@ -8426,6 +8426,8 @@ query_specification_start: > { > SELECT_LEX *sel; > LEX *lex= Lex; > + if (lex->sql_command == SQLCOM_INSERT) > + thd->get_stmt_da()->inc_current_row_for_warning(); > if (!(sel= lex->alloc_select(TRUE)) || lex->push_select(sel)) > MYSQL_YYABORT; better not to have an if() in this rule that's used almost for everything. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 8e21b91fd31: MDEV-26875: Wrong user in SET DEFAULT ROLE error
by Sergei Golubchik 11 Jan '22

11 Jan '22

Hi, Anel! On Jan 11, Anel Husakovic wrote: > revision-id: 8e21b91fd31 (mariadb-10.5.12-71-g8e21b91fd31) > parent(s): e5a9dcfda20 > author: Anel Husakovic > committer: Anel Husakovic > timestamp: 2021-10-25 14:43:38 +0200 > message: > > MDEV-26875: Wrong user in SET DEFAULT ROLE error > > diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc > index 65f053aa09b..0a59f5fce37 100644 > --- a/sql/sql_acl.cc > +++ b/sql/sql_acl.cc > @@ -3273,7 +3273,7 @@ static int check_user_can_set_role(THD *thd, const char *user, > { > /* Role is not granted but current user can see the role */ > my_printf_error(ER_INVALID_ROLE, "User %`s@%`s has not been granted role %`s", > - MYF(0), thd->security_ctx->priv_user, > + MYF(0), user, > thd->security_ctx->priv_host, rolename); That's strange. you'll use `user` and `thd->security_ctx->priv_host` ? They generally correspond to two different accounts. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] b14b6f31fa2: MDEV-26843: Inconsistent behavior of ROW_NUMBER upon resignalling from function Analysis: ROW_NUMBER should be 0 because the error produced is not related to statement that modifies table data. It is just wrong number of arguments. However, ROW_NUMBER is 1 because m_current_row_for_warning is not set to 0 before executing function. Fix: Set m_current_row_for_warning to 0.
by Sergei Golubchik 11 Jan '22

11 Jan '22

Hi, Rucha! What if you have INSERT t1 VALUES (f1(1)), (f1(2)), (f1()); ? On Jan 11, Rucha Deodhar wrote: > revision-id: b14b6f31fa2 (mariadb-10.6.1-143-gb14b6f31fa2) > parent(s): d555ae38381 > author: Rucha Deodhar > committer: Rucha Deodhar > timestamp: 2021-10-21 17:13:13 +0530 > message: > > MDEV-26843: Inconsistent behavior of ROW_NUMBER upon resignalling from > function > Analysis: ROW_NUMBER should be 0 because the error produced is not related to > statement that modifies table data. It is just wrong number of arguments. > However, ROW_NUMBER is 1 because m_current_row_for_warning is not set to 0 > before executing function. > Fix: Set m_current_row_for_warning to 0. > > diff --git a/sql/sp_head.cc b/sql/sp_head.cc > index d1cf5c2d9bd..ebae2f356bd 100644 > --- a/sql/sp_head.cc > +++ b/sql/sp_head.cc > @@ -2033,6 +2033,7 @@ sp_head::execute_function(THD *thd, Item **argp, uint argcount, > */ > if (argcount != m_pcont->context_var_count()) > { > + thd->get_stmt_da()->reset_current_row_for_warning(0); > /* > Need to use my_error here, or it will not terminate the > invoking query properly. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 394bb653b0b: MDEV-26695: Number of an invalid row is not calculated for table value constructor
by Sergei Golubchik 11 Jan '22

11 Jan '22

Hi, Rucha! please, see below On Jan 11, Rucha Deodhar wrote: > revision-id: 394bb653b0b (mariadb-10.6.1-140-g394bb653b0b) > parent(s): 3c857a07d92 > author: Rucha Deodhar > committer: Rucha Deodhar > timestamp: 2021-10-20 01:30:04 +0530 > message: > > MDEV-26695: Number of an invalid row is not calculated for table value > constructor > > Analysis: counter does not increment while sending rows for table value > constructor and so row_number assumes the default value (0 in this case). > Fix: Increment the counter to avoid counter using default value. > > diff --git a/sql/sql_tvc.cc b/sql/sql_tvc.cc > index 49f319b3856..a191ec9c328 100644 > --- a/sql/sql_tvc.cc > +++ b/sql/sql_tvc.cc > @@ -422,7 +422,9 @@ bool table_value_constr::exec(SELECT_LEX *sl) > DBUG_ENTER("table_value_constr::exec"); > List_iterator_fast<List_item> li(lists_of_values); > List_item *elem; > + THD *cur_thd= sl->parent_lex->thd; > ha_rows send_records= 0; > + int rc; > > if (select_options & SELECT_DESCRIBE) > DBUG_RETURN(false); > @@ -438,16 +440,21 @@ bool table_value_constr::exec(SELECT_LEX *sl) > > while ((elem= li++)) > { > + cur_thd->get_stmt_da()->inc_current_row_for_warning(); > if (send_records >= sl->master_unit()->lim.get_select_limit()) > - break; > + goto reset_counter_and_exit; Why goto, break worked here just fine, didn't it? Also, note that if you jump from here out of the loop, you'll do if (rc>0) where rc wasn't initialized yet. > - int rc= > - result->send_data_with_check(*elem, sl->master_unit(), send_records); > + rc= result->send_data_with_check(*elem, sl->master_unit(), send_records); > if (!rc) > send_records++; > else if (rc > 0) > - DBUG_RETURN(true); > + goto reset_counter_and_exit; break here too? > } > > +reset_counter_and_exit: > + cur_thd->get_stmt_da()->reset_current_row_for_warning(0); why do you need to reset here? > + if (rc>0) > + DBUG_RETURN(true); > + > if (result->send_eof()) > DBUG_RETURN(true); Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] ce69e0c8e78: MDEV-24920: Merge "old" SQL variable to "old_mode" sql variable
by Sergei Golubchik 10 Jan '22

10 Jan '22

Hi, Rucha! First, note that it's now a 10.9 task, so don't push it into 10.8, please. As for the patch - code-wise it's all good. New mode names are confusing and should be clarified, see the comment below. On Jan 10, Rucha Deodhar wrote: > revision-id: ce69e0c8e78 (mariadb-10.6.1-88-gce69e0c8e78) > parent(s): 22556499397 > author: Rucha Deodhar > committer: Rucha Deodhar > timestamp: 2021-10-06 13:25:29 +0530 > message: > > MDEV-24920: Merge "old" SQL variable to "old_mode" sql variable > > Analysis: There are 2 server variables- "old_mode" and "old". "old" is > no longer needed as "old_mode" has replaced it (however still used in > some places in the code). "old_mode" and "old" has same purpose- > emulate behavior from previous MariaDB versions. So they can be merged > to avoid confusion. Fix: Deprecate "old" variable and create another > mode for @@old_mode to mimic behavior of previous "old" variable. > Create specific modes for specifix task that --old sql variable was > doing earlier and use the new modes instead. > > diff --git a/mysql-test/suite/sys_vars/r/old_mode_basic.result b/mysql-test/suite/sys_vars/r/old_mode_basic.result > index a6b95f1c60c..2f3742160eb 100644 > --- a/mysql-test/suite/sys_vars/r/old_mode_basic.result > +++ b/mysql-test/suite/sys_vars/r/old_mode_basic.result > @@ -264,3 +264,42 @@ SET @@collation_database = @save_collation_database; > # > # End of 10.6 test > # > +# > +# Beginning of 10.7 test > +# 10.9 now > diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc > index c2a219ee015..d1eaacd777c 100644 > --- a/sql/sys_vars.cc > +++ b/sql/sys_vars.cc > @@ -3755,6 +3774,8 @@ static const char *old_mode_names[]= > "NO_PROGRESS_INFO", > "ZERO_DATE_TIME_CAST", > "UTF8_IS_UTF8MB3", > + "INDEX_MASKING", > + "CHECKSUM_FORMATTING", that's confusing. I don't understsand from the name of the option what "CHECKSUM_FORMATTING" or "INDEX_MASKING" changes. "NO_PROGRESS_INFO" is clear, there won't be progress info "UTF8_IS_UTF8MB3" is clear "ZERO_DATE_TIME_CAST" is kind of understandable too, CAST(<time> AS DATETIME) will be "0000-00-00 <time>", that is zero date CHECKSUM_FORMATTING and INDEX_MASKING is unclear. For CHECKSUM_FORMATTING I'd suggest CHECKSUM_SLOW_NULLS (see the commit that introduced this behavior: 496741d5761f) "INDEX_MASKING" was introduced in the commit 79542930ea1c, see the comment there, it explains what was done. This should help you to suggest a better name for this mode. > 0 > }; > > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 9ea85a70a75: MDEV-24654 GTID event falsely marked transactional
by Sergei Golubchik 10 Jan '22

10 Jan '22

Hi, Andrei! On Jan 10, Andrei Elkin wrote: > revision-id: 9ea85a70a75 (mariadb-10.2.40-4-g9ea85a70a75) > parent(s): 160d97a4aaa > author: Andrei Elkin > committer: Andrei Elkin > timestamp: 2021-08-08 14:20:57 +0300 > message: > > MDEV-24654 GTID event falsely marked transactional > > GTID event can be falsely marked transactional in few cases including > binary-logging on the slave side upon execution. > In some execution branches bin-logging of non-transactional > events can be done through transactional cache. E.g see comments > in THD::binlog_write_table_map(). I saw that and still couldn't understand why bin-logging of non-transactional events can be done through transactional cache. > In order to not create the false 'trans' tag the fact of Xid event > logging is checked to compute correct argument to Gtid_log_event > ctor. This is wrong. Transactions only end with a Xid event if all participating engines support XA. Try to create an InnoDB+FederatedX transaction (I just did) and it'll end with Query_log_event("COMMIT") > diff --git a/sql/log.cc b/sql/log.cc > index 1d9b4645421..7fef2e0d739 100644 > --- a/sql/log.cc > +++ b/sql/log.cc > @@ -8118,9 +8118,11 @@ MYSQL_BIN_LOG::write_transaction_or_stmt(group_commit_entry *entry, > uint64 commit_id) > { > binlog_cache_mngr *mngr= entry->cache_mngr; > + bool has_xid= entry->end_event->get_type_code() == XID_EVENT; > DBUG_ENTER("MYSQL_BIN_LOG::write_transaction_or_stmt"); > > - if (write_gtid_event(entry->thd, false, entry->using_trx_cache, commit_id)) > + if (write_gtid_event(entry->thd, false, > + entry->using_trx_cache && has_xid, commit_id)) > DBUG_RETURN(ER_ERROR_ON_WRITE); > > if (entry->using_stmt_cache && !mngr->stmt_cache.empty() && > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 734aee3cd3c: MDEV-27018 IF and COALESCE lose "json" property
by Sergei Golubchik 10 Jan '22

10 Jan '22

Hi, Alexander! On Dec 28, Alexander Barkov wrote: > revision-id: 734aee3cd3c (mariadb-10.5.13-33-g734aee3cd3c) > parent(s): 2776635cb98 > author: Alexander Barkov > committer: Alexander Barkov > timestamp: 2021-12-27 17:54:31 +0400 > message: > > MDEV-27018 IF and COALESCE lose "json" property > > Hybrid functions (IF, COALESCE, etc) did not preserve the JSON property > from their arguments. The same problem was repeatable for single row subselects. > > The problem happened because the method Item::is_json_type() was inconsistently > implemented across the Item hierarchy. For example, Item_hybrid_func > and Item_singlerow_subselect did not override is_json_type(). Okay. The approach is fine, but see some questions below: > diff --git a/sql/field.cc b/sql/field.cc > index 2c768527ced..8fa3bbd538c 100644 > --- a/sql/field.cc > +++ b/sql/field.cc > @@ -7277,6 +7277,19 @@ bool Field_longstr::send(Protocol *protocol) > } > > > +const Type_handler *Field_string::type_handler() const > +{ > + if (is_var_string()) > + return &type_handler_var_string; > + /* > + This is a temporary solution and will be fixed soon (in 10.9?). > + Type_handler_string_json will provide its own Field_string_json. why? > + */ > + if (Type_handler_json_common::has_json_valid_constraint(this)) could you please use names that don't depend on how exactly we detect json? especially if the goal is to move to FORMAT JSON syntax > + return &type_handler_string_json; > + return &type_handler_string; > +} > + > /* Copy a string and fill with space */ > > int Field_string::store(const char *from, size_t length,CHARSET_INFO *cs) > diff --git a/sql/sql_type.cc b/sql/sql_type.cc > index c1801c1ae3e..3b2753d80a6 100644 > --- a/sql/sql_type.cc > +++ b/sql/sql_type.cc > @@ -1755,19 +1755,110 @@ const Type_handler *Type_handler_typelib::cast_to_int_type_handler() const > > /***************************************************************************/ > > +class Recursive_type_pair_iterator > +{ > + const Type_handler *m_a; > + const Type_handler *m_b; > + uint m_switched_to_base_count; > +public: > + Recursive_type_pair_iterator(const Type_handler *a, > + const Type_handler *b, > + uint switched_to_base_count= 0) > + :m_a(a), m_b(b), m_switched_to_base_count(switched_to_base_count) > + { } > + const Type_handler *a() const { return m_a; } > + const Type_handler *b() const { return m_b; } > + Recursive_type_pair_iterator base() const > + { > + Recursive_type_pair_iterator res(m_a->type_handler_base(), > + m_b->type_handler_base()); > + res.m_switched_to_base_count= (res.m_a != NULL) + (res.m_b != NULL); > + if (res.m_a == NULL) > + res.m_a= m_a; > + if (res.m_b == NULL) > + res.m_b= m_b; > + return res; > + } > + bool done() const > + { > + switch (m_switched_to_base_count) > + { > + case 2: > + /* > + Expression: > + COALESCE(TYPE1, TYPE2) > + > + where both type handlers derive from some other handlers, e.g. > + VARCHAR -> TYPE1 > + TEXT -> TYPE2 > + > + Continue aggregation as: > + SELECT COALESCE(VARCHAR, TEXT) > + > + Note, we now have only one level of data type inheritance, > + e.g. VARCHAR -> VARCHAR/JSON > + > + This code branch will change when we have longer inheritance chains: > + A0 -> A1 -> A2 -> A3 > + B0 -> B1 -> B2 > + > + Functions like COALESE(A2, B2) will need to do full overload resolution: > + - iterate through all pairs from the below matrix > + - find all existing candidates > + - resolve ambiguities in case multiple candidates, etc. and I don't see how you're going to do that. you need to try m_a+m_b.base(), m_a.base()+m_b, and m_a.base()+m_b.base(). And that's not what you're doing. I suggest to add an assert that m_switched_to_base_count < 2 and stop trying to solve complex cases until we have at least one. > + > + A0 A1 A2 A3 > + B0 . . . . > + B1 . . . . > + B2 . . . . > + */ > + return false; > + > + case 1: > + /* > + Expression: > + COALESCE(TYPE1, TEXT) > + > + If only one handler derives from some other handler: > + VARCHAR-> TYPE1 > + > + Continue aggregation as: > + SELECT COALESCE(VARCHAR, TEXT) > + */ > + return false; > + > + case 0: > + default: > + /* > + Non of the two handlers are derived from other handlers. > + Nothing left to try. > + */ > + return true; > + } > + } > +}; > diff --git a/sql/sql_type_json.h b/sql/sql_type_json.h > index 6c4ee8cb2eb..4a394809a06 100644 > --- a/sql/sql_type_json.h > +++ b/sql/sql_type_json.h > @@ -21,18 +21,145 @@ ... > + bool Item_append_extended_type_info(Send_field_extended_metadata *to, > + const Item *item) const > + { > + return set_format_name(to); // Send "format=json" in the protocol > + } > + > + bool Item_hybrid_func_fix_attributes(THD *thd, > + const char *name, > + Type_handler_hybrid_field_type *hybrid, > + Type_all_attributes *attr, > + Item **items, uint nitems) > + const override > + { > + if (BASE::Item_hybrid_func_fix_attributes(thd, name, hybrid, attr, > + items, nitems)) > + return true; > + /* > + The above call can change the type handler on "hybrid", e.g. > + choose a proper BLOB type handler according to the calculated max_length. > + Convert general purpose string type handler to its JSON counterpart. > + This makes hybrid functions preserve JSON data types, e.g.: > + COALESCE(json_expr1, json_expr2) -> JSON > + */ > + hybrid->set_handler(json_type_handler_from_generic(hybrid->type_handler())); I don't get it. It's supposed to "fix attributes", like signed/unsigned, max length, etc. The correct type handler should've been set already, otherwise this type handler method wouldn't even be called. > + return false; > + } > }; > > + > +class Type_handler_string_json: > + public Type_handler_general_purpose_string_to_json<Type_handler_string, > + type_handler_string> > +{ }; > diff --git a/sql/sql_type_json.cc b/sql/sql_type_json.cc > index a804366ec03..a84f720bcc9 100644 > --- a/sql/sql_type_json.cc > +++ b/sql/sql_type_json.cc > @@ -20,17 +20,111 @@ > #include "sql_class.h" > > > -Type_handler_json_longtext type_handler_json_longtext; > +Named_type_handler<Type_handler_string_json> > + type_handler_string_json("char/json"); do you mean these names are not shown anywhere? > +Named_type_handler<Type_handler_varchar_json> > + type_handler_varchar_json("varchar/json"); > + > +Named_type_handler<Type_handler_tiny_blob_json> > + type_handler_tiny_blob_json("tinyblob/json"); > + > +Named_type_handler<Type_handler_blob_json> > + type_handler_blob_json("blob/json"); > + > +Named_type_handler<Type_handler_medium_blob_json> > + type_handler_medium_blob_json("mediumblob/json"); > + > +Named_type_handler<Type_handler_long_blob_json> > + type_handler_long_blob_json("longblob/json"); ... > +/* > + This method ressembles what Field_blob::type_handler() resembles (also below) > + does for general purpose BLOB type handlers. > +*/ > +const Type_handler * > +Type_handler_json_common::json_blob_type_handler_by_length_bytes(uint len) > +{ > + switch (len) { > + case 1: return &type_handler_tiny_blob_json; > + case 2: return &type_handler_blob_json; > + case 3: return &type_handler_medium_blob_json; > + } > + return &type_handler_long_blob_json; > +} Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 3

Re: [Maria-developers] ce507903d0c: MDEV-22742 UBSAN: Many overflow issues in strings/decimal.c - runtime error: signed integer overflow: x * y cannot be represented in type 'long long int' (on optimized builds).
by Sergei Golubchik 09 Jan '22

09 Jan '22

Hi, Alexey! On Jan 09, Alexey Botchkov wrote: > revision-id: ce507903d0c (mariadb-10.2.40-161-gce507903d0c) > parent(s): 0dae41637ab > author: Alexey Botchkov > committer: Alexey Botchkov > timestamp: 2021-11-22 09:58:46 +0400 > message: > > MDEV-22742 UBSAN: Many overflow issues in strings/decimal.c - runtime error: signed integer overflow: x * y cannot be represented in type 'long long int' (on optimized builds). > > Avoid integer overflow, do the check before the calculation. > > diff --git a/strings/decimal.c b/strings/decimal.c > index 9d18a9ce52a..6249d7e097a 100644 > --- a/strings/decimal.c > +++ b/strings/decimal.c > @@ -1128,13 +1128,16 @@ int decimal2ulonglong(const decimal_t *from, ulonglong *to) > > for (intg=from->intg; intg > 0; intg-=DIG_PER_DEC1) > { > - ulonglong y=x; > - x=x*DIG_BASE + *buf++; > - if (unlikely(y > ((ulonglong) ULONGLONG_MAX/DIG_BASE) || x < y)) > + if (unlikely ( > + x >= ULONGLONG_MAX/DIG_BASE && > + (x > ULONGLONG_MAX/DIG_BASE || > + *buf > (dec1) (ULONGLONG_MAX%DIG_BASE)))) This took me a while. Personally I find it easier to understand an exclusive condition, like x > ULONGLONG_MAX/DIG_BASE || (x == ULONGLONG_MAX/DIG_BASE && *buf > (dec1) (ULONGLONG_MAX%DIG_BASE)) but it's equivalent to your version, so ok to push. Did you check that this commit fixes all UBSAN issues mentioned in the MDEV-22742? Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 0dfe5a31573: MDEV-22441 implement a generic way to change a value of a variable in a scope
by Sergei Golubchik 09 Jan '22

09 Jan '22

Hi, Eugene! On Jan 09, Eugene Kosov wrote: > revision-id: 0dfe5a31573 (mariadb-10.7.1-13-g0dfe5a31573) > parent(s): 1e54a9716d2 > author: Eugene Kosov > committer: Eugene Kosov > timestamp: 2021-12-03 16:12:27 +0600 > message: > > MDEV-22441 implement a generic way to change a value of a variable in a scope > > Example: > { > auto _= make_scope_value(var, tmp_value); > } > > make_scope_value(): a function which returns RAII object which temporary > changes a value of a variable 1. Why have you not created a macro, like SCOPE_EXIT to avoid `auto _=` ? Particularly as one cannot use `_` to backup more than one variable. 2. better remove the new value from make_scope_value(). Creating a RAII backup and an assignment of a new value are two separate actions. The assignment can happen much later or conditionally or not at all. 3. (not part of MDEV-22441) See how SCOPE_EXIT is implemented in rocksdb, perhaps we should do it similarly? It's used like SCOPE_EXIT { mem_heap_free(heap); } Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 8a84f5a40c1: MDEV-24176 Preparations
by Sergei Golubchik 06 Jan '22

06 Jan '22

Hi, Aleksey! On Oct 17, Aleksey Midenkov wrote: > commit 8a84f5a40c1 > Author: Aleksey Midenkov <midenok(a)gmail.com> > Date: Thu May 27 17:00:14 2021 +0300 > > MDEV-24176 Preparations > > 1. moved fix_vcol_exprs() call to open_table() > > mysql_alter_table() doesn't do lock_tables() so it cannot win from > fix_vcol_exprs() from there. Tests affected: main.default_session This is likely wrong, but the old code was wrong too. Neither open_table nor lock_tables is called under LOCK TABLES, but the session environment can change, I suspect. > > 2. cleanup_excluding_fields_processor removed. > > That was just a quick hack to exclude wrongly working Item_field from > processing. Now it works due to correct execution environment (see > next commit). Related to MDEV-10355 does it work now, in that commit, or only after the next commit? what exactly do you mean by correct execution environment? > > 3. Vanilla cleanups and comments. > > diff --git a/sql/item.h b/sql/item.h > index cc1914a7ad4..7b7fe04f0b2 100644 > --- a/sql/item.h > +++ b/sql/item.h > @@ -2586,6 +2585,12 @@ class Item_ident :public Item_result_field > const char *db_name; > const char *table_name; > const char *field_name; > + /* > + NOTE: came from TABLE::alias_name_used and this is only a hint! It works > + only in need_correct_ident() condition. On other cases it is FALSE even if > + table_name is alias! It cannot be TRUE in these cases, lots of spaghetti > + logic depends on that. could you elaborate on that? > + */ > bool alias_name_used; /* true if item was resolved against alias */ > /* > Cached value of index for this field in table->field array, used by prep. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 5

Re: [Maria-developers] 3617add6975: MDEV-27208: Extend CRC32() and implement CRC32C()
by Sergei Golubchik 05 Jan '22

05 Jan '22

Hi, Marko! On Jan 03, Marko Mäkelä wrote: > revision-id: 3617add6975 (mariadb-10.6.1-252-g3617add6975) > parent(s): bc90f5d6967 > author: Marko Mäkelä > committer: Sergei Golubchik > timestamp: 2021-12-15 19:54:34 +0100 > message: > > MDEV-27208: Extend CRC32() and implement CRC32C() > > We used to define a native unary function crc32() that computes the CRC-32 > of a string using the ISO 3309 polynomial that is being used by zlib > and many others. Suggestions: 1. one class for both CRC functions, like class Item_func_crc32 :public Item_long_func { extern "C" unsigned (*crc_func)(unsigned int, const char *, size_t); and in the constructor crc_func= crc_variant ? my_crc32 : my_crc32c; there's no need to duplicate everything. 2. swap the arguments: CRC32(CRC32("Maria"), "DB") = CRC32("MariaDB") looks kind of better than CRC32("DB", CRC32("Maria")) = CRC32("MariaDB") Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 1

Re: [Maria-developers] 803b977229c: MDEV-26698: Incorrect row number upon INSERT .. SELECT from the same table: rows are counted twice
by Sergei Golubchik 31 Dec '21

31 Dec '21

Hi, Rucha! Ok to push. One suggestion below. On Dec 31, Rucha Deodhar wrote: > revision-id: 803b977229c (mariadb-10.2.40-192-g803b977229c) > parent(s): 42fea34d4a9 > author: Rucha Deodhar > committer: Rucha Deodhar > timestamp: 2021-12-29 16:09:10 +0530 > message: > > MDEV-26698: Incorrect row number upon INSERT .. SELECT from the same > table: rows are counted twice > > diff --git a/sql/sql_select.cc b/sql/sql_select.cc > index a331f4f3dbc..c8d5170f568 100644 > --- a/sql/sql_select.cc > +++ b/sql/sql_select.cc > @@ -18972,7 +18972,7 @@ evaluate_join_record(JOIN *join, JOIN_TAB *join_tab, > bool shortcut_for_distinct= join_tab->shortcut_for_distinct; > ha_rows found_records=join->found_records; > COND *select_cond= join_tab->select_cond; > - bool select_cond_result= TRUE; > + bool select_cond_result= TRUE, unlock_row= TRUE; > > DBUG_ENTER("evaluate_join_record"); > DBUG_PRINT("enter", > @@ -19124,6 +19124,7 @@ evaluate_join_record(JOIN *join, JOIN_TAB *join_tab, > > if (found) > { > + unlock_row= false; > enum enum_nested_loop_state rc; > /* A match from join_tab is found for the current partial join. */ > rc= (*join_tab->next_select)(join, join_tab+1, 0); > @@ -19147,11 +19148,6 @@ evaluate_join_record(JOIN *join, JOIN_TAB *join_tab, > if (shortcut_for_distinct && found_records != join->found_records) > DBUG_RETURN(NESTED_LOOP_NO_MORE_ROWS); instead of unlock_row() you can put DBUG_RETURN(NESTED_LOOP_OK); here. Might be a bit simpler to read. > } > - else > - { > - join->thd->get_stmt_da()->inc_current_row_for_warning(); > - join_tab->read_record.unlock_row(join_tab); > - } > } > else > { > @@ -19160,6 +19156,9 @@ evaluate_join_record(JOIN *join, JOIN_TAB *join_tab, > with the beginning coinciding with the current partial join. > */ > join->join_examined_rows++; > + } > + if (unlock_row) > + { > join->thd->get_stmt_da()->inc_current_row_for_warning(); > join_tab->read_record.unlock_row(join_tab); > } > @@ -26945,6 +26944,12 @@ AGGR_OP::end_send() > table->reginfo.lock_type= TL_UNLOCK; > > bool in_first_read= true; > + > + /* > + Reset the counter before copying rows from internal temporary table to > + INSERT table. > + */ > + join_tab->join->thd->get_stmt_da()->reset_current_row_for_warning(); > while (rc == NESTED_LOOP_OK) > { > int error; Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 4774601a3cd: MDEV-26698: Incorrect row number upon INSERT .. SELECT from the same table: rows are counted twice
by Sergei Golubchik 31 Dec '21

31 Dec '21

Hi, Rucha! On Dec 29, Rucha Deodhar wrote: > On Wed, Dec 29, 2021 at 12:50 AM Sergei Golubchik <serg(a)mariadb.org> wrote: > > > Why did put your fix here in a common code path in select, and not > > in, say, select_insert::send_data() ? > > Yes, the counter can be incremented in select_insert::send_data() too. > > But after writing record to INSERT table, we go back to > evaluate_join_records() where the counter is incremented > unconditionally. And CREATE TABLE...SELECT was failing with wrong > output because it also uses this code. > > I could have incremented counter conditionally so that it doesn't > increment twice like so: > > increment counter in select_insert::send_data() after record is written > if (thd->lex->sql_command == SQLCOM_INSERT_SELECT || > thd->lex->sql_command == SQLCOM_CREATE_TABLE) I'd think you wouldn't need this if(), as select_insert::send_data() is not used for other sql commands. > And increment counter in evalaute_join_records() after one nested loop > iteration ends (end_send()) > if (join->thd->lex->sql_command != SQLCOM_INSERT_SELECT && > join->thd->lex->sql_command != SQLCOM_CREATE_TABLE) I see. That's not good. > I had initially implemented it this way but it kinda looked hacky to me. > So I did a more general solution. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 4774601a3cd: MDEV-26698: Incorrect row number upon INSERT .. SELECT from the same table: rows are counted twice
by Sergei Golubchik 28 Dec '21

28 Dec '21

Hi, Rucha! Note test failures in buildbot, don't forget to update all result files before pushing. On Dec 28, Rucha Deodhar wrote: > revision-id: 4774601a3cd (mariadb-10.2.40-192-g4774601a3cd) > parent(s): 42fea34d4a9 > author: Rucha Deodhar > committer: Rucha Deodhar > timestamp: 2021-12-27 16:48:04 +0530 > message: > > MDEV-26698: Incorrect row number upon INSERT .. SELECT from the same > table: rows are counted twice > > Analysis: When the table we are trying to insert into and the SELECT table > are same for INSERT ... SELECT, rows from the SELECT table are copied into > internal temporary table and then to the INSERT table. We only want to > count the rows when we start inserting into the table. > Fix: Reset the counter to 1 before starting to copy from internal temporary > table to select table and then increment the counter. > > diff --git a/sql/sql_select.cc b/sql/sql_select.cc > index a331f4f3dbc..7a50f3d74ac 100644 > --- a/sql/sql_select.cc > +++ b/sql/sql_select.cc > @@ -3832,6 +3832,8 @@ mysql_select(THD *thd, > } > } > > + thd->get_stmt_da()->reset_current_row_for_warning(); What is this for? It wasn't needed before. > if ((err= join->optimize())) > { > goto err; // 1 > @@ -19124,10 +19126,10 @@ evaluate_join_record(JOIN *join, JOIN_TAB *join_tab, > > if (found) > { > + unlock_row= false; > enum enum_nested_loop_state rc; > /* A match from join_tab is found for the current partial join. */ > rc= (*join_tab->next_select)(join, join_tab+1, 0); > - join->thd->get_stmt_da()->inc_current_row_for_warning(); By moving this down you skip few returns, incl NESTED_LOOP_OK and NESTED_LOOP_NO_MORE_ROWS. Are you sure it's fine? Won't the counter be off, if the query execution continues after, say, NESTED_LOOP_OK? > if (rc != NESTED_LOOP_OK && rc != NESTED_LOOP_NO_MORE_ROWS) > DBUG_RETURN(rc); > if (return_tab < join->return_tab) > @@ -26945,6 +26943,12 @@ AGGR_OP::end_send() > table->reginfo.lock_type= TL_UNLOCK; > > bool in_first_read= true; > + > + /* > + Reset the counter before copying rows from internal temporary table to > + INSERT table. this code is also used for group by. Please, add a test for insert ... select ... group by Why did put your fix here in a common code path in select, and not in, say, select_insert::send_data() ? > + */ > + join_tab->join->thd->get_stmt_da()->reset_current_row_for_warning(); > while (rc == NESTED_LOOP_OK) > { > int error; > @@ -26968,6 +26972,8 @@ AGGR_OP::end_send() > else > { > rc= evaluate_join_record(join, join_tab, 0); > + /* Increment the counter after copying rows. */ > + join_tab->join->thd->get_stmt_da()->inc_current_row_for_warning(); > } > } Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

[Maria-developers] MDEV-26996: On removing SEL_ARG::is_ascending
by Sergey Petrunia 20 Dec '21

20 Dec '21

Hello Igor, Please check out this: https://github.com/MariaDB/server/tree/bb-10.8-mdev26996-no-sel-arg-ascendi… This is a patch that removes SEL_ARG::is_ascending. Does it make the code better in your opinion? BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://petrunia.net

1 0

Re: [Maria-developers] [Commits] 677643a: MDEV-27262 Unexpected index intersection with full index scan for an index
by Sergey Petrunia 20 Dec '21

20 Dec '21

Hello Igor, I'm trying examples with this patch: create table t1 (a int not null, key(a)); insert into t1 select seq from seq_1_to_1000; First, I debug this example: explain select * from t1 where a <10 or a>=10; The execution enters key_or(), passes through the new code this patch has added and key_or() returns NULL. Good. But if I modify the example slightly: explain select * from t1 where a <20 or a>=10 ; Here, key_or() still returns a SEL_ARG object that represents an infinite range: (gdb) fini Run till exit from #0 key_or (...) 0x0000555555f1fca2 in tree_or (...) Value returned is $24 = (SEL_ARG *) 0x7fff70064d88 (gdb) p dbug_print_sel_arg($24) $26 = 0x7fff70024390 "SEL_ARG(-inf<=a<=+inf)" Should this case be fixed as well? On Fri, Dec 17, 2021 at 02:11:40PM -0800, IgorBabaev wrote: > revision-id: 677643a80986107491b7886441f2828384f0494b (mariadb-10.2.31-1286-g677643a) > parent(s): 8bb55633699612279744c055e22eeca8d4058273 > author: Igor Babaev > committer: Igor Babaev > timestamp: 2021-12-17 14:11:39 -0800 > message: > > MDEV-27262 Unexpected index intersection with full index scan for an index > > If when extracting a range condition foran index from the WHERE condition I think the comment wording could be improved also. > Range Optimizer sees that the range condition covers the whole index then > such condition should be discarded because cannot it be used in any range > scan. In some cases Range Optimizer really does it, but there remained some > conditions for which it was not done. As a result the optimizer could > produce index merge plans with the full index scan for one of the indexes > participating in the index merge. > This could be observed in one of the test cases from index_merge1.inc > where a plan with index_merge_sort_union was produced and in the test case > reported for this bug where a plan with index_merge_sort_intersect was > produced. In both cases one of two index scans participating in index merge > ran over the whole index. > The patch slightly changes the original above mentioned test case from > index_merge1.inc to be able to produce an intended plan employing > index_merge_sort_union. The original query was left to show that index > merge is not used for it anymore. > It should be noted that for the plan with index_merge_sort_intersect could > be chosen for execution only due to a defect in the InnoDB code that > returns wrong estimates for the cardinality of big ranges. > > This bug led to serious problems in 10.4+ where the optimization using > Rowid filters is employed (see mdev-26446). > > Approved by Oleksandr Byelkin <sanja(a)mariadb.com> > > --- > mysql-test/include/index_merge1.inc | 8 +++- > mysql-test/r/index_merge_myisam.result | 12 +++-- > mysql-test/r/range_innodb.result | 81 ++++++++++++++++++++++++++++++++++ > mysql-test/t/range_innodb.test | 78 ++++++++++++++++++++++++++++++++ > sql/opt_range.cc | 7 +++ > 5 files changed, 181 insertions(+), 5 deletions(-) > > diff --git a/mysql-test/include/index_merge1.inc b/mysql-test/include/index_merge1.inc > index b168a76..440f1f7 100644 > --- a/mysql-test/include/index_merge1.inc > +++ b/mysql-test/include/index_merge1.inc > @@ -150,15 +150,19 @@ explain select * from t0 where > (((key3 <7 and key7 < 6) or key5 < 2) and (key5 < 5 or key6 < 6)); > > explain select * from t0 where > - ((key3 <5 or key5 < 4) and (key1 < 4 or key2 < 4)) > + ((key3 < 4 or key5 < 4) and (key1 < 4 or key2 < 4)) > or > ((key3 >=5 or key5 < 2) and (key5 < 5 or key6 < 6)); > > explain select * from t0 force index(i1, i2, i3, i4, i5, i6 ) where > - ((key3 <5 or key5 < 4) and (key1 < 4 or key2 < 4)) > + ((key3 < 4 or key5 < 4) and (key1 < 4 or key2 < 4)) > or > ((key3 >=5 or key5 < 2) and (key5 < 5 or key6 < 6)); > > +explain select * from t0 force index(i1, i2, i3, i4, i5, i6 ) where > + ((key3 < 5 or key5 < 4) and (key1 < 4 or key2 < 4)) > + or > + ((key3 >=5 or key5 < 2) and (key5 < 5 or key6 < 6)); > # 8. Verify that "order by" after index merge uses filesort > select * from t0 where key1 < 5 or key8 < 4 order by key1; > > diff --git a/mysql-test/r/index_merge_myisam.result b/mysql-test/r/index_merge_myisam.result > index 5a23092..a096c34 100644 > --- a/mysql-test/r/index_merge_myisam.result > +++ b/mysql-test/r/index_merge_myisam.result > @@ -173,17 +173,23 @@ or > id select_type table type possible_keys key key_len ref rows Extra > 1 SIMPLE t0 index_merge i1,i2,i3,i5,i6,i7 i3,i5 4,4 NULL 11 Using sort_union(i3,i5); Using where > explain select * from t0 where > -((key3 <5 or key5 < 4) and (key1 < 4 or key2 < 4)) > +((key3 < 4 or key5 < 4) and (key1 < 4 or key2 < 4)) > or > ((key3 >=5 or key5 < 2) and (key5 < 5 or key6 < 6)); > id select_type table type possible_keys key key_len ref rows Extra > 1 SIMPLE t0 ALL i1,i2,i3,i5,i6 NULL NULL NULL 1024 Using where > explain select * from t0 force index(i1, i2, i3, i4, i5, i6 ) where > -((key3 <5 or key5 < 4) and (key1 < 4 or key2 < 4)) > +((key3 < 4 or key5 < 4) and (key1 < 4 or key2 < 4)) > or > ((key3 >=5 or key5 < 2) and (key5 < 5 or key6 < 6)); > id select_type table type possible_keys key key_len ref rows Extra > -1 SIMPLE t0 index_merge i1,i2,i3,i5,i6 i3,i5 0,4 NULL 1024 Using sort_union(i3,i5); Using where > +1 SIMPLE t0 index_merge i1,i2,i3,i5,i6 i3,i5 4,4 NULL 1024 Using sort_union(i3,i5); Using where > +explain select * from t0 force index(i1, i2, i3, i4, i5, i6 ) where > +((key3 < 5 or key5 < 4) and (key1 < 4 or key2 < 4)) > +or > +((key3 >=5 or key5 < 2) and (key5 < 5 or key6 < 6)); > +id select_type table type possible_keys key key_len ref rows Extra > +1 SIMPLE t0 ALL i1,i2,i3,i5,i6 NULL NULL NULL 1024 Using where > select * from t0 where key1 < 5 or key8 < 4 order by key1; > key1 key2 key3 key4 key5 key6 key7 key8 > 1 1 1 1 1 1 1 1023 > diff --git a/mysql-test/r/range_innodb.result b/mysql-test/r/range_innodb.result > index f2349f2..ccb6da3 100644 > --- a/mysql-test/r/range_innodb.result > +++ b/mysql-test/r/range_innodb.result > @@ -108,3 +108,84 @@ DROP TABLE t0,t1; > SET @@GLOBAL.debug_dbug = @saved_dbug; > set @@optimizer_switch= @optimizer_switch_save; > # End of 10.1 tests > +# > +# MDEV-27262: Index intersection with full scan over an index > +# > +CREATE TABLE t1 ( > +id int(10) unsigned NOT NULL AUTO_INCREMENT, > +p char(32) DEFAULT NULL, > +es tinyint(3) unsigned NOT NULL DEFAULT 0, > +er tinyint(3) unsigned NOT NULL DEFAULT 0, > +x mediumint(8) unsigned NOT NULL DEFAULT 0, > +PRIMARY KEY (id), > +INDEX es (es), > +INDEX x (x), > +INDEX er (er,x), > +INDEX p (p) > +) ENGINE=InnoDB DEFAULT CHARSET=latin1; > +insert into t1(es,er) select 0, 1 from seq_1_to_45; > +insert into t1(es,er) select 0, 2 from seq_1_to_49; > +insert into t1(es,er) select 0, 3 from seq_1_to_951; > +insert into t1(es,er) select 0, 3 from seq_1_to_1054; > +insert into t1(es,er) select 0, 6 from seq_1_to_25; > +insert into t1(es,er) select 0, 11 from seq_1_to_1; > +insert into t1(es,er) select 1, 1 from seq_1_to_45; > +insert into t1(es,er) select 1, 2 from seq_1_to_16; > +insert into t1(es,er) select 1, 3 from seq_1_to_511; > +insert into t1(es,er) select 1, 4 from seq_1_to_687; > +insert into t1(es,er) select 1, 6 from seq_1_to_50; > +insert into t1(es,er) select 1, 7 from seq_1_to_4; > +insert into t1(es,er) select 1, 11 from seq_1_to_1; > +insert into t1(es,er) select 2, 1 from seq_1_to_82; > +insert into t1(es,er) select 2, 2 from seq_1_to_82; > +insert into t1(es,er) select 2, 3 from seq_1_to_1626; > +insert into t1(es,er) select 2, 4 from seq_1_to_977; > +insert into t1(es,er) select 2, 6 from seq_1_to_33; > +insert into t1(es,er) select 2, 11 from seq_1_to_1; > +insert into t1(es,er) select 3, 1 from seq_1_to_245; > +insert into t1(es,er) select 3, 2 from seq_1_to_81; > +insert into t1(es,er) select 3, 3 from seq_1_to_852; > +insert into t1(es,er) select 3, 4 from seq_1_to_2243; > +insert into t1(es,er) select 3, 6 from seq_1_to_44; > +insert into t1(es,er) select 3, 11 from seq_1_to_1; > +insert into t1(es,er) select 4, 1 from seq_1_to_91; > +insert into t1(es,er) select 4, 2 from seq_1_to_83; > +insert into t1(es,er) select 4, 3 from seq_1_to_297; > +insert into t1(es,er) select 4, 4 from seq_1_to_2456; > +insert into t1(es,er) select 4, 6 from seq_1_to_19; > +insert into t1(es,er) select 4, 11 from seq_1_to_1; > +update t1 set p='foobar'; > +update t1 set x=0; > +set @save_isp=@@innodb_stats_persistent; > +set global innodb_stats_persistent= 1; > +analyze table t1; > +Table Op Msg_type Msg_text > +test.t1 analyze status OK > +set optimizer_switch='index_merge_sort_intersection=on'; > +SELECT * FROM t1 > +WHERE ((p = 'foo' AND er != 4) OR er = 4 ) AND (es >= 4) LIMIT 2; > +id p es er x > +14645 foobar 4 4 0 > +14646 foobar 4 4 0 > +EXPLAIN EXTENDED SELECT * FROM t1 > +WHERE ((p = 'foo' AND er != 4) OR er = 4 ) AND (es >= 4) LIMIT 2; > +id select_type table type possible_keys key key_len ref rows filtered Extra > +1 SIMPLE t1 range es,er,p es 1 NULL # 100.00 Using index condition; Using where > +Warnings: > +Note 1003 select `test`.`t1`.`id` AS `id`,`test`.`t1`.`p` AS `p`,`test`.`t1`.`es` AS `es`,`test`.`t1`.`er` AS `er`,`test`.`t1`.`x` AS `x` from `test`.`t1` where (`test`.`t1`.`p` = 'foo' and `test`.`t1`.`er` <> 4 or `test`.`t1`.`er` = 4) and `test`.`t1`.`es` >= 4 limit 2 > +set optimizer_switch='index_merge_sort_intersection=off'; > +SELECT * FROM t1 > +WHERE ((p = 'foo' AND er != 4) OR er = 4 ) AND (es >= 4) LIMIT 2; > +id p es er x > +14645 foobar 4 4 0 > +14646 foobar 4 4 0 > +EXPLAIN EXTENDED SELECT * FROM t1 > +WHERE ((p = 'foo' AND er != 4) OR er = 4 ) AND (es >= 4) LIMIT 2; > +id select_type table type possible_keys key key_len ref rows filtered Extra > +1 SIMPLE t1 range es,er,p es 1 NULL # 100.00 Using index condition; Using where > +Warnings: > +Note 1003 select `test`.`t1`.`id` AS `id`,`test`.`t1`.`p` AS `p`,`test`.`t1`.`es` AS `es`,`test`.`t1`.`er` AS `er`,`test`.`t1`.`x` AS `x` from `test`.`t1` where (`test`.`t1`.`p` = 'foo' and `test`.`t1`.`er` <> 4 or `test`.`t1`.`er` = 4) and `test`.`t1`.`es` >= 4 limit 2 > +set optimizer_switch='index_merge_sort_intersection=default'; > +set global innodb_stats_persistent= @save_isp; > +DROP TABLE t1; > +# End of 10.2 tests > diff --git a/mysql-test/t/range_innodb.test b/mysql-test/t/range_innodb.test > index 428e5c2..7420e72 100644 > --- a/mysql-test/t/range_innodb.test > +++ b/mysql-test/t/range_innodb.test > @@ -116,3 +116,81 @@ SET @@GLOBAL.debug_dbug = @saved_dbug; > set @@optimizer_switch= @optimizer_switch_save; > > --echo # End of 10.1 tests > + > +--echo # > +--echo # MDEV-27262: Index intersection with full scan over an index > +--echo # > + > +--source include/have_sequence.inc > + > +CREATE TABLE t1 ( > + id int(10) unsigned NOT NULL AUTO_INCREMENT, > + p char(32) DEFAULT NULL, > + es tinyint(3) unsigned NOT NULL DEFAULT 0, > + er tinyint(3) unsigned NOT NULL DEFAULT 0, > + x mediumint(8) unsigned NOT NULL DEFAULT 0, > + PRIMARY KEY (id), > + INDEX es (es), > + INDEX x (x), > + INDEX er (er,x), > + INDEX p (p) > +) ENGINE=InnoDB DEFAULT CHARSET=latin1; > + > +insert into t1(es,er) select 0, 1 from seq_1_to_45; > +insert into t1(es,er) select 0, 2 from seq_1_to_49; > +insert into t1(es,er) select 0, 3 from seq_1_to_951; > +insert into t1(es,er) select 0, 3 from seq_1_to_1054; > +insert into t1(es,er) select 0, 6 from seq_1_to_25; > +insert into t1(es,er) select 0, 11 from seq_1_to_1; > +insert into t1(es,er) select 1, 1 from seq_1_to_45; > +insert into t1(es,er) select 1, 2 from seq_1_to_16; > +insert into t1(es,er) select 1, 3 from seq_1_to_511; > +insert into t1(es,er) select 1, 4 from seq_1_to_687; > +insert into t1(es,er) select 1, 6 from seq_1_to_50; > +insert into t1(es,er) select 1, 7 from seq_1_to_4; > +insert into t1(es,er) select 1, 11 from seq_1_to_1; > +insert into t1(es,er) select 2, 1 from seq_1_to_82; > +insert into t1(es,er) select 2, 2 from seq_1_to_82; > +insert into t1(es,er) select 2, 3 from seq_1_to_1626; > +insert into t1(es,er) select 2, 4 from seq_1_to_977; > +insert into t1(es,er) select 2, 6 from seq_1_to_33; > +insert into t1(es,er) select 2, 11 from seq_1_to_1; > +insert into t1(es,er) select 3, 1 from seq_1_to_245; > +insert into t1(es,er) select 3, 2 from seq_1_to_81; > +insert into t1(es,er) select 3, 3 from seq_1_to_852; > +insert into t1(es,er) select 3, 4 from seq_1_to_2243; > +insert into t1(es,er) select 3, 6 from seq_1_to_44; > +insert into t1(es,er) select 3, 11 from seq_1_to_1; > +insert into t1(es,er) select 4, 1 from seq_1_to_91; > +insert into t1(es,er) select 4, 2 from seq_1_to_83; > +insert into t1(es,er) select 4, 3 from seq_1_to_297; > +insert into t1(es,er) select 4, 4 from seq_1_to_2456; > +insert into t1(es,er) select 4, 6 from seq_1_to_19; > +insert into t1(es,er) select 4, 11 from seq_1_to_1; > +update t1 set p='foobar'; > +update t1 set x=0; > +set @save_isp=@@innodb_stats_persistent; > +set global innodb_stats_persistent= 1; > +analyze table t1; > + > +let $q= > +SELECT * FROM t1 > + WHERE ((p = 'foo' AND er != 4) OR er = 4 ) AND (es >= 4) LIMIT 2; > + > +set optimizer_switch='index_merge_sort_intersection=on'; > +eval $q; > +--replace_column 9 # > +eval EXPLAIN EXTENDED $q; > + > +set optimizer_switch='index_merge_sort_intersection=off'; > +# execution of $q and explain for it led to an assertion failure in 10.4 > +# (with the optimizer switch rowid_filter set to 'on') > +eval $q; > +--replace_column 9 # > +eval EXPLAIN EXTENDED $q; > +set optimizer_switch='index_merge_sort_intersection=default'; > + > +set global innodb_stats_persistent= @save_isp; > +DROP TABLE t1; > + > +--echo # End of 10.2 tests > diff --git a/sql/opt_range.cc b/sql/opt_range.cc > index f3f1843..2e05b88 100644 > --- a/sql/opt_range.cc > +++ b/sql/opt_range.cc > @@ -9413,6 +9413,13 @@ key_or(RANGE_OPT_PARAM *param, SEL_ARG *key1,SEL_ARG *key2) > key2->copy_min(tmp); > if (!(key1=key1->tree_delete(tmp))) > { // Only one key in tree > + if (key2->min_flag & NO_MIN_RANGE && > + key2->max_flag & NO_MAX_RANGE) > + { > + if (key2->maybe_flag) > + return new SEL_ARG(SEL_ARG::MAYBE_KEY); > + return 0; // Always true OR > + } > key1=key2; > key1->make_root(); > key2=key2_next; BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://petrunia.net

1 0

[Maria-developers] Review: bb-10.8-wlad-MDEV-26713
by Alexander Barkov 13 Dec '21

13 Dec '21

Hi Wlad, You've done a great piece of work. Thanks and congrats. I generally like all the changes. Have a few comments only, please see below. commit 1411d5f40ceaf2eecabbcf68588743439003f4a2 Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Fri Nov 19 11:42:46 2021 +0100 Windows : incorrect handling of non-ASCIIs in get_tty_password Why no MDEV? + wchar_t wbuf[80]; + char to[80*3]; /* there is at most 3 bytes per wchar, in any codepage */ + wchar_t *pos=wbuf,*end=wbuf + array_elements(wbuf)-1; Isn't it "at most 4 bytes per char"? ############################################################# commit 7af691ca11c95a385aff4889a56870d70843e78d Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Fri Nov 19 12:03:48 2021 +0100 MDEV-27090: Windows client - ReadConsoleA does not work correctly with UTF8 codepage Corresponding Windows bug https://github.com/microsoft/terminal/issues/4551 Use ReadConsoleW instead and convert to console's input codepage, to workaround. Also, disable VT sequences in the console output, as we do not knows what type of data comes with SELECT, we do not want VT escapes there. Remove my_cgets() + while (nchars > 0) + { + if (wstrbuf[nchars - 1] != '\n' && wstrbuf[nchars - 1] != '\r') + break; + wstrbuf[--nchars]= 0; + } + + wstrbuf[nchars]= 0; The inner assignment wstrbuf[--nchars]=0 inside every iteration is not needed. I'd write it this way: for ( ; nchars > 0 ; nchars--) { if (wstrbuf[nchars - 1] != '\n' && wstrbuf[nchars - 1] != '\r') break; } wstrbuf[nchars]= 0; But the outer assignment to 0 does not seem to be needed either. Why do it? my_convert() does not need the trailing zero. It uses only sizes passed. So this should be enough: + strbuf[len]= 0; - pthread_auto_mutex_decl(my_conio_cs); - - /* lock the console for the current process*/ - if (pthread_auto_mutex_lock(my_conio_cs, GetCurrentProcessId(), INFINITE)) - { There was a mutex lock inside my_cgets(). Why was it needed before the patch and why is it not needed any more? ##################################################### commit 5c09e9ce4f922fe9674f3f85329742c0c5498a28 Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Fri Nov 19 14:03:51 2021 +0100 MDEV-27092 Windows - services that have non-ASCII characters do not work with activeCodePage=UTF8 CreateServiceA, OpenServiceA, and couple of other functions do not work correctly with non-ASCII character, in the special case where application has defined activeCodePage=UTF8. ... - Copyright (c) 2011, 2012, Monty Program Ab + Copyright (c) 2011, 2021 Monty Program Ab MariaDB Corporation? This code: + AWSTRDUP(w_ServiceName,lpServiceName); + AWSTRDUP(w_DisplayName,lpDisplayName); + AWSTRDUP(w_BinaryPathName, lpBinaryPathName); + AWSTRDUP(w_LoadOrderGroup, lpLoadOrderGroup); + AWSTRDUP(w_Dependencies, lpDependencies); + AWSTRDUP(w_ServiceStartName, lpServiceStartName); + AWSTRDUP(w_Password, lpPassword); + + free(w_ServiceName); + free(w_DisplayName); + free(w_BinaryPathName); + free(w_LoadOrderGroup); + free(w_Dependencies); + free(w_ServiceStartName); + free(w_Password); duplicates two times. Consider wrapping these variables into a class and reuse the class in both cases. ########################################################## commit d4056a28ec5ae0c9a0c3144a8e90357dc574723f Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Fri Nov 19 14:14:38 2021 +0100 MDEV-27093 Do not pass root password in clear text from mariadb-install-db.exe to bootstrap Pass the hash instead, so that bootstrap would not misinterpret it for string with different charset. The password was passed using hex encoding. But the comment says it was clear text. Wrong comment? I did not understand the change. Reading the MDEV did not make it clearer. What was wrong? How does the patch fix the problem? Can we discuss on slack? ######################################################## commit fa29a64de918d2045df3cce1d4a546de5597c0bd Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Mon Nov 22 12:29:15 2021 +0100 MDEV-26713 set activeCodePage=UTF8 for windows programs - Use corresponding entry in the manifest, as described in https://docs.microsoft.com/en-us/windows/apps/design/globalizing/use-utf8-c… - Set console codepage(s) to UTF8, if ANSI codepage is UTF8 This effectively causes UTF8 to be used as default client encoding, for Windows 1903 and later - Allow UTF8 on the command line for MTR, for the affected Windows versions Looks OK for me. ######################################################## commit b88d54132369b938ba041c0d3f97a5a212210b37 Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Mon Nov 22 12:34:10 2021 +0100 MDEV-26713 : Treat codepage 65001 as utf8mb4, not just utf8 Also, fix the "UTF8" option in MSI, which is responsible for character-set-server setting Looks OK for me. ######################################################## commit b9c38470ed6e04185f7f84a056034b7254336d51 Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Mon Nov 22 13:22:05 2021 +0100 MDEV-26713 utf8 support on Windows, mysql_upgrade_service part Handle upgrade - on Windows that is capable of UTF8 codepage, convert entries in my.ini from ANSI to UTF8, during upgrade Reason is that paths must now be utf8. For the client , user name, database etc should be in UTF8, too, as this is now the default charset. Should not we add an UTF8 BOM to the beginning of the file, so text editors do not break it later. The patch looks too large for me. It seems it does more than the comments above tell. Looks like convenience refactoring, but it's hard to follow. It would be easier to understand changes if you split it into two parts: 1. the refactoring that adds functions like upgrade_config_file(), fix_section(), is_mariadb_section(), fix_and_check_datadir(), I guess this refactoring is a non-functional change that should not change the behaviour, right? 2. the actual conversion part on top of it ######################################################## commit 0700393b570c6b9a7b4347d3381a7cf05d1499ba Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Tue Nov 23 12:28:28 2021 +0100 mysqltest, Windows - support background execution in 'exec' command If last character in command is ampersand, do not use my_popen/my_pclose in "exec", instead do CreateProcess() without a window. I suggest you add an MDEV for this change and explain in there why we did not need this feature earlier and why we need it now and how it relates to MDEV-26713. ######################################################## commit 7d50d83b3bcab5a3be761f98722e33ed0b01ab23 Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Tue Nov 23 13:05:25 2021 +0100 MDEV-26713 utf8 support on Windows , add mysql_install_db tests Add mysql_install_db test with some i18n, for data dir and root password Looks OK for me. ######################################################## commit b2cc09fe76f526bd4cd76131a52263b13b3828a5 Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Wed Nov 24 10:15:11 2021 +0100 MDEV-26713 Add test for mysql_install_db creating service, with i18 Looks OK for me. ######################################################## commit 51ce069376fec45d340563a4f7630f8a155afd64 Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Fri Nov 26 18:41:35 2021 +0100 MDEV-26713 Introduce --password-codepage client setting I'm not sure we need this new options. Looks like over-engineering a bit. Is the scenario it covers really deserves documenting, learning, etc? I'd better disallow non-ASCII passwords in the future, and right now just add a warning: You're using a non-ASCII password, which is not portable. If you have problems connecting to the database, please ask the administrator to change the password. ######################################################## commit b155c4036cfb3ffdaa7c272048ac9d32e3182d78 Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Mon Nov 29 19:47:36 2021 +0100 MDEV-26713 set console codepage to what user set in --default-character-set I prefer to keep the comment that you removed: - if (!fs_cset_cache) - { - char buf[10]= "cp"; - GetLocaleInfo(LOCALE_SYSTEM_DEFAULT, LOCALE_IDEFAULTANSICODEPAGE, - buf+2, sizeof(buf)-3); - /* - We cannot call get_charset_by_name here - because fs_character_set() is executed before - LOCK_THD_charset mutex initialization, which - is used inside get_charset_by_name. - As we're now interested in cp932 only, - let's just detect it using strcmp(). - */ - fs_cset_cache= - #ifdef HAVE_CHARSET_cp932 - !strcmp(buf, "cp932") ? &my_charset_cp932_japanese_ci : - #endif - &my_charset_bin; This function uses both default_charset and charset_info->cs_name.str: +static void adjust_console_codepage() +{ +#ifdef _WIN32 + const char *name= charset_info->cs_name.str; + if (my_set_console_cp(default_charset) < 0) And it seems the former is set to the latter just before the function call: default_charset= (char *)charset_info->cs_name.str; <------ HERE put_info("Charset changed", INFO_INFO); adjust_console_codepage(); I find this hard to follow. Can you please add a parameter to adjust_console_codepage() and pass either charset_info->cs_name.str (or default_charset) when calling? Also, I think this function should only be called when default_charset was some explicit name, not MYSQL_AUTODETECT_CHARSET_NAME. I think the logic should be: 1. If --default-character-set=auto, then detect it from the console. 2. If --default-character-set=csname, then set the console CP to csname. Now in case of N1 it seems to detect the character set from the console, then sets the console to what has been detected. Is the last step really needed in case of auto-detection? ######################################################## commit b38c341cc392630c798eb754f87338d37ec95cdf Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Sun Dec 5 17:08:18 2021 +0100 Fix race condition in test. Do not change console codepage in a possibly shared console. Start new console for chcp. Can you extend the comment and tell what actually happened before this change? Greetings.

2 2

[Maria-developers] Review for: MDEV-26938 Support descending indexes internally in InnoDB (server part)
by Sergey Petrunia 11 Dec '21

11 Dec '21

Hi Serg, There is some non-trivial input too, after all. Please find below. > commit da2903f03de039693354176fda836e6642d6d0f0 > Author: Sergei Golubchik <serg(a)mariadb.org> > Date: Wed Nov 24 16:50:21 2021 +0100 > > MDEV-26938 Support descending indexes internally in InnoDB (server part) > > * preserve DESC index property in the parser > * store it in the frm (only for HA_KEY_ALG_BTREE) > * read it from the frm > * show it in SHOW CREATE > * skip DESC indexes in opt_range.cc and opt_sum.cc > * ORDER BY test > == Trivial input == > diff --git a/sql/unireg.cc b/sql/unireg.cc > index 2726b9a68c2..d6449eeca4c 100644 > --- a/sql/unireg.cc > +++ b/sql/unireg.cc > @@ -685,6 +685,14 @@ static uint pack_keys(uchar *keybuff, uint key_count, KEY *keyinfo, > DBUG_PRINT("loop", ("flags: %lu key_parts: %d key_part: %p", > key->flags, key->user_defined_key_parts, > key->key_part)); > + /* For SPATIAL, FULLTEXT and HASH indexes (anything other than B-tree), > + ignore the ASC/DESC attribute of columns. */ > + const uchar ha_reverse_sort= key->algorithm == HA_KEY_ALG_BTREE > + ? HA_REVERSE_SORT : > + (key->algorithm == HA_KEY_ALG_UNDEF && > + !(key->flags & (HA_FULLTEXT | HA_SPATIAL))) > + ? HA_REVERSE_SORT : 0; The above is hard to read for me. I think if-statement form would be more readable: const uchar ha_reverse_sort= 0; if (key->algorithm == HA_KEY_ALG_BTREE || (key->algorithm == HA_KEY_ALG_UNDEF && !(key->flags & (HA_FULLTEXT | HA_SPATIAL)) ha_reverse_sort= HA_REVERSE_SORT; == Issue #1: ordered index scan is not handled in ha_partition == Testcase: create table t10 (a int, b int, key(a desc)) partition by hash(a) partitions 4; insert into t10 select seq, seq from seq_0_to_999; MariaDB [test]> select * from t10 order by a limit 3; +------+------+ | a | b | +------+------+ | 3 | 3 | | 7 | 7 | | 11 | 11 | +------+------+ 3 rows in set (0.002 sec) Correct output: MariaDB [test]> select * from t10 use index() order by a limit 3; +------+------+ | a | b | +------+------+ | 0 | 0 | | 1 | 1 | | 2 | 2 | +------+------+ 3 rows in set (0.011 sec) == Issue #2: extra warning == MariaDB [test]> create table t1 (a int, b int, key(a), key(a)); Query OK, 0 rows affected, 1 warning (0.016 sec) MariaDB [test]> show warnings; +-------+------+--------------------------------------------------------------------------------------+ | Level | Code | Message | +-------+------+--------------------------------------------------------------------------------------+ | Note | 1831 | Duplicate index `a_2`. This is deprecated and will be disallowed in a future release | +-------+------+--------------------------------------------------------------------------------------+ 1 row in set (0.000 sec) This is ok. MariaDB [test]> create table t2 (a int, b int, key(a), key(a desc)); Query OK, 0 rows affected, 1 warning (0.004 sec) MariaDB [test]> show warnings; +-------+------+--------------------------------------------------------------------------------------+ | Level | Code | Message | +-------+------+--------------------------------------------------------------------------------------+ | Note | 1831 | Duplicate index `a_2`. This is deprecated and will be disallowed in a future release | +-------+------+--------------------------------------------------------------------------------------+ 1 row in set (0.000 sec) This is probably not? MySQL doesn't produce this warning in this case. BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://petrunia.net

2 3

[Maria-developers] Review: bb-10.8-wlad-MDEV-26713
by Alexander Barkov 10 Dec '21

10 Dec '21

Hi Wlad, You've done a great piece of work. Thanks and congrats. I generally like all the changes. Have a few comments only, please see below. commit 1411d5f40ceaf2eecabbcf68588743439003f4a2 Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Fri Nov 19 11:42:46 2021 +0100 Windows : incorrect handling of non-ASCIIs in get_tty_password Why no MDEV? + wchar_t wbuf[80]; + char to[80*3]; /* there is at most 3 bytes per wchar, in any codepage */ + wchar_t *pos=wbuf,*end=wbuf + array_elements(wbuf)-1; Isn't it "at most 4 bytes per char"? ############################################################# commit 7af691ca11c95a385aff4889a56870d70843e78d Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Fri Nov 19 12:03:48 2021 +0100 MDEV-27090: Windows client - ReadConsoleA does not work correctly with UTF8 codepage Corresponding Windows bug https://github.com/microsoft/terminal/issues/4551 Use ReadConsoleW instead and convert to console's input codepage, to workaround. Also, disable VT sequences in the console output, as we do not knows what type of data comes with SELECT, we do not want VT escapes there. Remove my_cgets() + while (nchars > 0) + { + if (wstrbuf[nchars - 1] != '\n' && wstrbuf[nchars - 1] != '\r') + break; + wstrbuf[--nchars]= 0; + } + + wstrbuf[nchars]= 0; The inner assignment wstrbuf[--nchars]=0 inside every iteration is not needed. I'd write it this way: for ( ; nchars > 0 ; nchars--) { if (wstrbuf[nchars - 1] != '\n' && wstrbuf[nchars - 1] != '\r') break; } wstrbuf[nchars]= 0; But the outer assignment to 0 does not seem to be needed either. Why do it? my_convert() does not need the trailing zero. It uses only sizes passed. So this should be enough: + strbuf[len]= 0; - pthread_auto_mutex_decl(my_conio_cs); - - /* lock the console for the current process*/ - if (pthread_auto_mutex_lock(my_conio_cs, GetCurrentProcessId(), INFINITE)) - { There was a mutex lock inside my_cgets(). Why was it needed before the patch and why is it not needed any more? ##################################################### commit 5c09e9ce4f922fe9674f3f85329742c0c5498a28 Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Fri Nov 19 14:03:51 2021 +0100 MDEV-27092 Windows - services that have non-ASCII characters do not work with activeCodePage=UTF8 CreateServiceA, OpenServiceA, and couple of other functions do not work correctly with non-ASCII character, in the special case where application has defined activeCodePage=UTF8. ... - Copyright (c) 2011, 2012, Monty Program Ab + Copyright (c) 2011, 2021 Monty Program Ab MariaDB Corporation? This code: + AWSTRDUP(w_ServiceName,lpServiceName); + AWSTRDUP(w_DisplayName,lpDisplayName); + AWSTRDUP(w_BinaryPathName, lpBinaryPathName); + AWSTRDUP(w_LoadOrderGroup, lpLoadOrderGroup); + AWSTRDUP(w_Dependencies, lpDependencies); + AWSTRDUP(w_ServiceStartName, lpServiceStartName); + AWSTRDUP(w_Password, lpPassword); + + free(w_ServiceName); + free(w_DisplayName); + free(w_BinaryPathName); + free(w_LoadOrderGroup); + free(w_Dependencies); + free(w_ServiceStartName); + free(w_Password); duplicates two times. Consider wrapping these variables into a class and reuse the class in both cases. ########################################################## commit d4056a28ec5ae0c9a0c3144a8e90357dc574723f Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Fri Nov 19 14:14:38 2021 +0100 MDEV-27093 Do not pass root password in clear text from mariadb-install-db.exe to bootstrap Pass the hash instead, so that bootstrap would not misinterpret it for string with different charset. The password was passed using hex encoding. But the comment says it was clear text. Wrong comment? I did not understand the change. Reading the MDEV did not make it clearer. What was wrong? How does the patch fix the problem? Can we discuss on slack? ######################################################## commit fa29a64de918d2045df3cce1d4a546de5597c0bd Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Mon Nov 22 12:29:15 2021 +0100 MDEV-26713 set activeCodePage=UTF8 for windows programs - Use corresponding entry in the manifest, as described in https://docs.microsoft.com/en-us/windows/apps/design/globalizing/use-utf8-c… - Set console codepage(s) to UTF8, if ANSI codepage is UTF8 This effectively causes UTF8 to be used as default client encoding, for Windows 1903 and later - Allow UTF8 on the command line for MTR, for the affected Windows versions Looks OK for me. ######################################################## commit b88d54132369b938ba041c0d3f97a5a212210b37 Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Mon Nov 22 12:34:10 2021 +0100 MDEV-26713 : Treat codepage 65001 as utf8mb4, not just utf8 Also, fix the "UTF8" option in MSI, which is responsible for character-set-server setting Looks OK for me. ######################################################## commit b9c38470ed6e04185f7f84a056034b7254336d51 Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Mon Nov 22 13:22:05 2021 +0100 MDEV-26713 utf8 support on Windows, mysql_upgrade_service part Handle upgrade - on Windows that is capable of UTF8 codepage, convert entries in my.ini from ANSI to UTF8, during upgrade Reason is that paths must now be utf8. For the client , user name, database etc should be in UTF8, too, as this is now the default charset. Should not we add an UTF8 BOM to the beginning of the file, so text editors do not break it later. The patch looks too large for me. It seems it does more than the comments above tell. Looks like convenience refactoring, but it's hard to follow. It would be easier to understand changes if you split it into two parts: 1. the refactoring that adds functions like upgrade_config_file(), fix_section(), is_mariadb_section(), fix_and_check_datadir(), I guess this refactoring is a non-functional change that should not change the behaviour, right? 2. the actual conversion part on top of it ######################################################## commit 0700393b570c6b9a7b4347d3381a7cf05d1499ba Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Tue Nov 23 12:28:28 2021 +0100 mysqltest, Windows - support background execution in 'exec' command If last character in command is ampersand, do not use my_popen/my_pclose in "exec", instead do CreateProcess() without a window. I suggest you add an MDEV for this change and explain in there why we did not need this feature earlier and why we need it now and how it relates to MDEV-26713. ######################################################## commit 7d50d83b3bcab5a3be761f98722e33ed0b01ab23 Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Tue Nov 23 13:05:25 2021 +0100 MDEV-26713 utf8 support on Windows , add mysql_install_db tests Add mysql_install_db test with some i18n, for data dir and root password Looks OK for me. ######################################################## commit b2cc09fe76f526bd4cd76131a52263b13b3828a5 Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Wed Nov 24 10:15:11 2021 +0100 MDEV-26713 Add test for mysql_install_db creating service, with i18 Looks OK for me. ######################################################## commit 51ce069376fec45d340563a4f7630f8a155afd64 Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Fri Nov 26 18:41:35 2021 +0100 MDEV-26713 Introduce --password-codepage client setting I'm not sure we need this new options. Looks like over-engineering a bit. Is the scenario it covers really deserves documenting, learning, etc? I'd better disallow non-ASCII passwords in the future, and right now just add a warning: You're using a non-ASCII password, which is not portable. If you have problems connecting to the database, please ask the administrator to change the password. ######################################################## commit b155c4036cfb3ffdaa7c272048ac9d32e3182d78 Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Mon Nov 29 19:47:36 2021 +0100 MDEV-26713 set console codepage to what user set in --default-character-set I prefer to keep the comment that you removed: - if (!fs_cset_cache) - { - char buf[10]= "cp"; - GetLocaleInfo(LOCALE_SYSTEM_DEFAULT, LOCALE_IDEFAULTANSICODEPAGE, - buf+2, sizeof(buf)-3); - /* - We cannot call get_charset_by_name here - because fs_character_set() is executed before - LOCK_THD_charset mutex initialization, which - is used inside get_charset_by_name. - As we're now interested in cp932 only, - let's just detect it using strcmp(). - */ - fs_cset_cache= - #ifdef HAVE_CHARSET_cp932 - !strcmp(buf, "cp932") ? &my_charset_cp932_japanese_ci : - #endif - &my_charset_bin; This function uses both default_charset and charset_info->cs_name.str: +static void adjust_console_codepage() +{ +#ifdef _WIN32 + const char *name= charset_info->cs_name.str; + if (my_set_console_cp(default_charset) < 0) And it seems the former is set to the latter just before the function call: default_charset= (char *)charset_info->cs_name.str; <------ HERE put_info("Charset changed", INFO_INFO); adjust_console_codepage(); I find this hard to follow. Can you please add a parameter to adjust_console_codepage() and pass either charset_info->cs_name.str (or default_charset) when calling? Also, I think this function should only be called when default_charset was some explicit name, not MYSQL_AUTODETECT_CHARSET_NAME. I think the logic should be: 1. If --default-character-set=auto, then detect it from the console. 2. If --default-character-set=csname, then set the console CP to csname. Now in case of N1 it seems to detect the character set from the console, then sets the console to what has been detected. Is the last step really needed in case of auto-detection? ######################################################## commit b38c341cc392630c798eb754f87338d37ec95cdf Author: Vladislav Vaintroub <wlad(a)mariadb.com> Date: Sun Dec 5 17:08:18 2021 +0100 Fix race condition in test. Do not change console codepage in a possibly shared console. Start new console for chcp. Can you extend the comment and tell what actually happened before this change? Greetings.

1 0

Re: [Maria-developers] b95c5105e28: MDEV-17554 Auto-create new partition for system versioned tables with history partitioned by INTERVAL/LIMIT
by Sergei Golubchik 07 Dec '21

07 Dec '21

Hi, Aleksey! This is a review of diff b95c5105e2 2552bce607 This was quite good, thanks! Just a couple of questions/comments, see below. On Nov 23, Aleksey Midenkov wrote: > diff --git a/mysql-test/suite/versioning/t/partition.test b/mysql-test/suite/versioning/t/partition.test > index 006a65e1a16..a5bb8426ae4 100644 > --- a/mysql-test/suite/versioning/t/partition.test > +++ b/mysql-test/suite/versioning/t/partition.test > @@ -1070,15 +1080,572 @@ alter table t1 add partition partitions 2; ... > +--let $datadir= `select @@datadir` > +--let $dummy= $datadir/test/t1#P#p1.ibd > +--write_file $dummy > +EOF > + > +set timestamp= unix_timestamp('2000-01-01 01:00:00'); > +--error ER_GET_ERRNO > +update t1 set x= x + 2; That's a good one! > +show warnings; > +--remove_file $dummy > + > diff --git a/sql/table.h b/sql/table.h > index 2e074abcea0..d861db261bf 100644 > --- a/sql/table.h > +++ b/sql/table.h > @@ -910,6 +911,11 @@ struct TABLE_SHARE > vers_kind_t versioned; > period_info_t vers; > period_info_t period; > + /* > + Protect multiple threads from repeating partition auto-create over > + single share. could you please add here (to the comment) TODO remove it when partitioning metadata will be in TABLE_SHARE > + */ > + bool vers_skip_auto_create; > > bool init_period_from_extra2(period_info_t *period, const uchar *data, > const uchar *end); > @@ -2557,6 +2567,11 @@ struct TABLE_LIST > bool merged; > bool merged_for_insert; > bool sequence; /* Part of NEXTVAL/CURVAL/LASTVAL */ > + /* > + Protect single thread from repeating partition auto-create over > + multiple share instances (as the share is closed on backoff action). > + */ > + bool vers_skip_create; I don't understand it. What does it mean "multiple share instances"? Can there be multiple TABLE_SHARE objects for the same table? How? > > /* > Items created by create_view_field and collected to change them in case > diff --git a/sql/sql_partition.cc b/sql/sql_partition.cc > index d7bb02bbd4e..56d5a68efd0 100644 > --- a/sql/sql_partition.cc > +++ b/sql/sql_partition.cc > @@ -2587,11 +2587,15 @@ char *generate_partition_syntax(THD *thd, partition_info *part_info, > err+= str.append(ctime, ctime_len); > err+= str.append('\''); > } > + if (vers_info->auto_hist) > + err+= str.append(STRING_WITH_LEN(" AUTO")); > } > - if (vers_info->limit) > + else if (vers_info->limit) > { > err+= str.append(STRING_WITH_LEN("LIMIT ")); > err+= str.append_ulonglong(vers_info->limit); > + if (vers_info->auto_hist) > + err+= str.append(STRING_WITH_LEN(" AUTO")); Can you do this str.appent("AUTO") once, after the if()? Not in both branches? > } > } > else if (part_info->part_expr) > diff --git a/sql/partition_info.cc b/sql/partition_info.cc > index fd92e437cac..ddcc6f413ba 100644 > --- a/sql/partition_info.cc > +++ b/sql/partition_info.cc > @@ -865,9 +870,167 @@ void partition_info::vers_set_hist_part(THD *thd) > { > vers_info->hist_part= next; > if (next->range_value > thd->query_start()) > - return; > + { > + error= false; > + break; > + } > + } > + if (error) > + { > + if (auto_hist) > + { > + *create_count= 0; > + const my_time_t hist_end= (my_time_t) vers_info->hist_part->range_value; > + DBUG_ASSERT(thd->query_start() >= hist_end); > + MYSQL_TIME h0, q0; > + my_tz_OFFSET0->gmt_sec_to_TIME(&h0, hist_end); > + my_tz_OFFSET0->gmt_sec_to_TIME(&q0, thd->query_start()); > + longlong q= pack_time(&q0); > + longlong h= pack_time(&h0); > + while (h <= q) > + { > + if (date_add_interval(thd, &h0, vers_info->interval.type, > + vers_info->interval.step)) > + return true; > + h= pack_time(&h0); > + ++*create_count; > + if (*create_count == MAX_PARTITIONS - 2) > + { > + my_error(ER_TOO_MANY_PARTITIONS_ERROR, MYF(ME_WARNING)); > + my_error(ER_VERS_HIST_PART_FAILED, MYF(0), > + table->s->db.str, table->s->table_name.str); > + return true; > + } > + } > + } > + else > + { > + DBUG_ASSERT(!vers_info->auto_hist); ^^^ This assert fails for me in the update-big test > + my_error(WARN_VERS_PART_FULL, MYF(ME_WARNING|ME_ERROR_LOG), > + table->s->db.str, table->s->table_name.str, > + vers_info->hist_part->partition_name, "INTERVAL"); > + } > + } > + } > + > + return false; > +} > + > + > diff --git a/sql/sql_base.cc b/sql/sql_base.cc > index ac22a63bdba..459edb614f7 100644 > --- a/sql/sql_base.cc > +++ b/sql/sql_base.cc > @@ -4203,6 +4444,15 @@ bool open_tables(THD *thd, const DDL_options_st &options, > } > > thd->current_tablenr= 0; > + > +#ifdef WITH_PARTITION_STORAGE_ENGINE > + if (!thd->stmt_arena->is_conventional()) > + { > + for (tables= *start; tables; tables= tables->next_global) > + tables->vers_skip_create= false; > + } This looks a bit like an overkill, you do it for every prepare, while auto-adding a partition is a rare event, as you said. May be you can do instead in TABLE::vers_switch_partition uint *create_count= !thd->stmt_arena->is_conventional() && table_list->vers_skip_create ? > +#endif > + > restart: > /* > Close HANDLER tables which are marked for flush or against which there Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 3

[Maria-developers] some questions for the community
by 杨凡 26 Nov '21

26 Nov '21

Hello, I am a student of Xi'an University of Posts and Telecommunications. My name is yangfan.I want to ask something about community code sharing and GSOC. I am now in my sophomore year, and I am very interested in databases, especially your marria database. I am now familiar with the c/c++ language and master the basic Linux system programming and network programming. Currently learning C++ multi-threaded programming. I can use basic SQL statements, databases, basic Linux commands, and some knowledge about operating systems and networks. I joined the xiyou linux group. Now, I want to learn quickly and contribute code to your community. Can you tell me what to do? And I want to participate in GSOC in 2022. If you can give me some help, I will be very happy. I only learned about open source culture from university. After a long period of study and thinking, I feel that open source is a very mea ningful thing. It not only helps personal growth but also helps a product iteration. The slogan of our group is free sharing and openness. At the same time, I continue to promote open source culture. I am fascinated by this culture. I also want to join a community and make my contribution to you. Although my current ability is not very strong, but I will continue to work hard. I am very interested in programming, and I want to focus on distributed and database work now and in the future. Maybe my English is not very good, I hope you can be more tolerant.

1 0

Re: [Maria-developers] a5fca9a6e30: MENT-651 [6/8] store cache managers in a list
by Sergei Golubchik 26 Nov '21

26 Nov '21

Hi, Nikita! On Nov 14, Nikita Malyavin wrote: > revision-id: a5fca9a6e30 (mariadb-10.5.2-478-ga5fca9a6e30) > parent(s): ad6e7b87107 > author: Nikita Malyavin > committer: Nikita Malyavin > timestamp: 2021-01-27 17:28:05 +1000 > message: > > MENT-651 [6/8] store cache managers in a list again, a couple of lines with a description would be good here > diff --git a/mysql-test/suite/binlog/t/online_alter.test b/mysql-test/suite/binlog/t/online_alter.test > index 804e847d008..4ed2db74bd6 100644 > --- a/mysql-test/suite/binlog/t/online_alter.test > +++ b/mysql-test/suite/binlog/t/online_alter.test > @@ -218,7 +219,7 @@ set autocommit = 0; > start transaction; > update t1 set b= 7007 where a = 7; > --error ER_DUP_ENTRY > -update t1 set a= 8 where a = 8 or a = 9; > +update t1 set a= 8, b= 8008 where a = 8 or a = 9 order by a; why did you change tests for what is, as far as I understand, just an optimization? > commit; > set autocommit = 1; > > diff --git a/sql/handler.cc b/sql/handler.cc > index 8b95653e69f..8b22167e729 100644 > --- a/sql/handler.cc > +++ b/sql/handler.cc > @@ -6660,7 +6660,16 @@ static int binlog_log_row_online_alter(TABLE* table, > THD *thd= table->in_use; > > if (!table->online_alter_cache) > - table->online_alter_cache= thd->binlog_setup_cache_data(); > + { > + auto *cache_mngr= online_alter_binlog_get_cache_mngr(thd, table); > + // Use transaction cache directly, if it is not multi-transaction mode > + table->online_alter_cache= binlog_get_cache_data(cache_mngr, > + !thd->in_multi_stmt_transaction_mode()); > + > + trans_register_ha(thd, false, binlog_hton, 0); > + if (thd->in_multi_stmt_transaction_mode()) > + trans_register_ha(thd, true, binlog_hton, 0); I still don't understand that. Why do you need a handlerton, why do you fake a transaction? And why do you need to search a list in THD, instead of storing cache_mngr in TABLE_SHARE? > + } > > // We need to log all columns for the case if alter table changes primary key. > table->use_all_columns(); > diff --git a/sql/log.cc b/sql/log.cc > index 6c678624230..c58096e6d9a 100644 > --- a/sql/log.cc > +++ b/sql/log.cc > @@ -2258,14 +2298,18 @@ static int binlog_rollback(handlerton *hton, THD *thd, bool all) > { > DBUG_ENTER("binlog_rollback"); > > - for (TABLE *table= thd->open_tables; table; table= table->next) > - { > - if (!table->online_alter_cache) > - continue; > - table->online_alter_cache->reset(); > - delete table->online_alter_cache; > - table->online_alter_cache= NULL; > - } > + bool is_ending_trans= ending_trans(thd, all); > + > + /* > + This is a crucial moment that we are running through > + thd->online_alter_cache_list, and not through thd->open_tables to cleanup > + stmt cache, though both have it. The reason is that the tables can be closed > + to that moment in case of an error. > + The same reason applies to the fact we don't store cache_mngr in the table > + itself -- because it can happen to be not existing. > + Still in case if tables are left opened > + */ > + binlog_online_alter_cleanup(thd->online_alter_cache_list, is_ending_trans); still don't understand that comment either :( Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 2

[Maria-developers] Squash aaed3436 MENT-651 [4/8] ALTER ONLINE TABLE
by Nikita Malyavin 25 Nov '21

25 Nov '21

Sergei, I think all the commits starting from [4/8] ALTER ONLINE TABLE should be squashed. I will also add a better description in the commit message covering our latest discussions. I would also embed a reference to the original unsquashed branch. It seems to me as a good idea in sense of helping the reader investigate the code -- Yours truly, Nikita Malyavin

2 5

Re: [Maria-developers] 4af7a583802: Refactor MYSQL_BIN_LOG: extract Event_log ancestor
by Sergei Golubchik 24 Nov '21

24 Nov '21

Hi, Nikita! On Nov 12, Nikita Malyavin wrote: > revision-id: 4af7a583802 (mariadb-10.5.2-474-g4af7a583802) > parent(s): 2da3dc5d422 > author: Nikita Malyavin > committer: Nikita Malyavin > timestamp: 2021-01-08 20:59:09 +1000 > message: > > Refactor MYSQL_BIN_LOG: extract Event_log ancestor Would be good to have some explanation here. I understand that you'll use this refactoring in following commits, but conceptually, what Event_log is? I mean, like Event_log is a log where Log_event_xxx events are written, but not _something else that MYSQL_BIN_LOG is and Event_log isn't_ or Event_log is a log where Log_event_xxx events are written. While MYSQL_BIN_LOG is Event_log that also does .... > diff --git a/sql/log.cc b/sql/log.cc > index 827a34c2883..6f4e3191588 100644 > --- a/sql/log.cc > +++ b/sql/log.cc > @@ -3701,64 +3765,25 @@ bool MYSQL_BIN_LOG::open(const char *log_name, > write_file_name_to_index_file= 1; > } a bit strange that MYSQL_BIN_LOG::open doesn't use Event_log::open > diff --git a/sql/log.h b/sql/log.h > index 73cd66d5a4a..74c409e1ac7 100644 > --- a/sql/log.h > +++ b/sql/log.h > @@ -327,6 +327,7 @@ class MYSQL_LOG > bool strip_ext, char *buff); > virtual int generate_new_name(char *new_name, const char *log_name, > ulong next_log_number); > + inline mysql_mutex_t* get_log_lock() { return &LOCK_log; } > protected: > /* LOCK_log is inited by init_pthread_objects() */ > mysql_mutex_t LOCK_log; > @@ -376,6 +377,64 @@ struct Rows_event_factory > } > }; > > +class Event_log: public MYSQL_LOG > +{ > +public: > +#if !defined(MYSQL_CLIENT) > + Rows_log_event* > + prepare_pending_rows_event(THD *thd, TABLE* table, > + binlog_cache_data *cache_data, > + uint32 serv_id, size_t needed, > + bool is_transactional, > + Rows_event_factory event_factory); forgot to delete the same from MYSQL_BIN_LOG > +#endif > + > + bool flush_and_sync(bool *synced); same > + void cleanup() > + { > + if (inited) > + mysql_mutex_destroy(&LOCK_binlog_end_pos); > + > + MYSQL_LOG::cleanup(); > + } > + void init_pthread_objects() > + { > + MYSQL_LOG::init_pthread_objects(); > + > + mysql_mutex_init(m_key_LOCK_binlog_end_pos, &LOCK_binlog_end_pos, > + MY_MUTEX_INIT_SLOW); > + } > + > + bool open( > + const char *log_name, > + enum_log_type log_type, > + const char *new_name, ulong next_file_number, > + enum cache_type io_cache_type_arg); > + IO_CACHE *get_log_file() { return &log_file; } > + > + int write_description_event(enum_binlog_checksum_alg checksum_alg, > + bool encrypt, bool dont_set_created); > + > + bool write_event(Log_event *ev, binlog_cache_data *data, IO_CACHE *file); > +}; > + > /* Tell the io thread if we can delay the master info sync. */ > #define SEMI_SYNC_SLAVE_DELAY_SYNC 1 > /* Tell the io thread if the current event needs a ack. */ > @@ -451,7 +510,7 @@ class binlog_cache_data; > struct rpl_gtid; > struct wait_for_commit; > > -class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG > +class MYSQL_BIN_LOG: public TC_LOG, private Event_log > { > /** The instrumentation key to use for @ LOCK_index. */ > PSI_mutex_key m_key_LOCK_index; > @@ -849,15 +901,13 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG > > - bool write_event(Log_event *ev, binlog_cache_data *data, IO_CACHE *file); > + bool write_event(Log_event *ev, binlog_cache_data *data, IO_CACHE *file) > + { return Event_log::write_event(ev, data, file); } Perhaps using Event_log::write_event(Log_event *ev, binlog_cache_data *data, IO_CACHE *file); ? > bool write_event(Log_event *ev) { return write_event(ev, 0, &log_file); } > > bool write_event_buffer(uchar* buf,uint len); > @@ -918,7 +968,6 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG > uint next_file_id(); > inline char* get_index_fname() { return index_file_name;} > inline char* get_log_fname() { return log_file_name; } > - inline char* get_name() { return name; } > inline mysql_mutex_t* get_log_lock() { return &LOCK_log; } should remove that too, I reckon > inline mysql_cond_t* get_bin_log_cond() { return &COND_bin_log_updated; } > inline IO_CACHE* get_log_file() { return &log_file; } Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 3

Re: [Maria-developers] f3603cbfaeb: report progress
by Sergei Golubchik 24 Nov '21

24 Nov '21

Hi, Nikita! On Nov 15, Nikita Malyavin wrote: > revision-id: f3603cbfaeb (mariadb-10.5.2-483-gf3603cbfaeb) > parent(s): 3b6038bee88 > author: Nikita Malyavin > committer: Nikita Malyavin > timestamp: 2021-01-29 03:47:52 +1000 > message: > > report progress Perhaps, squash this with your commit "test progress reporting" ? > diff --git a/sql/sql_table.cc b/sql/sql_table.cc > index 29d0f487303..87dbe226c2e 100644 > --- a/sql/sql_table.cc > +++ b/sql/sql_table.cc > @@ -11300,6 +11300,9 @@ static int online_alter_read_from_binlog(THD *thd, rpl_group_info *rgi, > > IO_CACHE *log_file= log->flip(); > > + thd_progress_report(thd, 0, my_b_write_tell(log_file) > + / rgi->tables_to_lock->m_conv_table->s->reclength); 1. why do you do this "/ rgi->...->s->reclength" ? 2. How does it work with your "cache flipping"? While data is copied between tables, concurrent connections write into an IO_CACHE. Then you flip, they what, write into the second IO_CACHE? What file length will you use here in thd_progress_report()? > + > do > { > const auto *descr_event= rgi->rli->relay_log.description_event_for_exec; > @@ -11319,6 +11322,9 @@ static int online_alter_read_from_binlog(THD *thd, rpl_group_info *rgi, > free_root(&event_mem_root, MYF(MY_KEEP_PREALLOC)); > if (ev != rgi->rli->relay_log.description_event_for_exec) > delete ev; > + thd_progress_report(thd, my_b_tell(log_file) > + / rgi->tables_to_lock->m_conv_table->s->reclength, > + thd->progress.max_counter); > } > while(!error); Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 3

Re: [Maria-developers] 9277b838aad: Forbid savepoints setup while ONLINE ALTER goes on
by Sergei Golubchik 24 Nov '21

24 Nov '21

Hi, Nikita! On Nov 15, Nikita Malyavin wrote: > revision-id: 9277b838aad (mariadb-10.5.2-485-g9277b838aad) > parent(s): 37b6f5cb8ef > author: Nikita Malyavin > committer: Nikita Malyavin > timestamp: 2021-01-29 03:47:52 +1000 > message: > > Forbid savepoints setup while ONLINE ALTER goes on Why is that? Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 2

Re: [Maria-developers] 903ae1c03ae: Fix running without binlog
by Sergei Golubchik 24 Nov '21

24 Nov '21

Hi, Nikita! On Nov 15, Nikita Malyavin wrote: > revision-id: 903ae1c03ae (mariadb-10.5.2-479-g903ae1c03ae) > parent(s): a5fca9a6e30 > author: Nikita Malyavin > committer: Nikita Malyavin > timestamp: 2021-01-27 17:28:05 +1000 > message: > > Fix running without binlog Strange. With such a comment I'd expect you add a test for ALTER TABLE with disabled binlog. Instead you change an error to a warning in one of the existing tests. This test change doesn't match the commit comment. > diff --git a/mysql-test/suite/binlog/r/online_alter.result b/mysql-test/suite/binlog/r/online_alter.result > index 9bca94b8a87..bb3fa85e76c 100644 > --- a/mysql-test/suite/binlog/r/online_alter.result > +++ b/mysql-test/suite/binlog/r/online_alter.result > @@ -34,9 +34,9 @@ insert into t1 values (123), (456), (789); > set debug_sync= 'now SIGNAL end'; > connection default; > Warnings: > -Error 1364 Field 'b' doesn't have a default value > -Error 1364 Field 'b' doesn't have a default value > -Error 1364 Field 'b' doesn't have a default value > +Warning 1364 Field 'b' doesn't have a default value > +Warning 1364 Field 'b' doesn't have a default value > +Warning 1364 Field 'b' doesn't have a default value > select * from t1; > a b > 5 0 > diff --git a/sql/sql_table.cc b/sql/sql_table.cc > index f0d1167ad61..e348056646d 100644 > --- a/sql/sql_table.cc > +++ b/sql/sql_table.cc > @@ -11308,9 +11308,12 @@ static int online_alter_read_from_binlog(THD *thd, rpl_group_info *rgi, > break; > > ev->thd= thd; > + bool abort_on_warning= thd->abort_on_warning; > + thd->abort_on_warning= false; > thd->set_n_backup_active_arena(&event_arena, &backup_arena); > error= ev->apply_event(rgi); > thd->restore_active_arena(&event_arena, &backup_arena); > + thd->abort_on_warning= abort_on_warning; > > event_arena.free_items(); > free_root(&event_mem_root, MYF(MY_KEEP_PREALLOC)); > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 2

[Maria-developers] Review for: MDEV-27036: Add asserts to detect duplicate JSON keys
by Sergey Petrunia 24 Nov '21

24 Nov '21

Hi Sergei, Please find my input below. > diff --git a/sql/sql_select.cc b/sql/sql_select.cc > index 0dfe95e81b0..fa07608177a 100644 > --- a/sql/sql_select.cc > +++ b/sql/sql_select.cc > @@ -7910,28 +7910,27 @@ best_access_path(JOIN *join, > can make an adjustment is a special case of the criteria used > in ReuseRangeEstimateForRef-3. > */ > - if (table->opt_range_keys.is_set(key) && > + auto use_range_estimates= table->opt_range_keys.is_set(key) && > (const_part & What is the reason for this change? Please restore the old code. > (((key_part_map)1 << table->opt_range[key].key_parts)-1)) == > (((key_part_map)1 << table->opt_range[key].key_parts)-1) && > table->opt_range[key].ranges == 1 && > - records > (double) table->opt_range[key].rows) > + records > (double) table->opt_range[key].rows; > + if (use_range_estimates) > { > records= (double) table->opt_range[key].rows; > trace_access_idx.add("used_range_estimates", "clipped down"); > } > else > { > + trace_access_idx.add("used_range_estimates",false); > if (table->opt_range_keys.is_set(key)) > { > - trace_access_idx.add("used_range_estimates",false) > - .add("cause", > - "not better than ref estimates"); > + cause= "not better than ref estimates"; As we have already discussed: Do not assign this to "cause". To avoid duplicate names, change the name above from "cause" to e.g. "reason". > } > else > { > - trace_access_idx.add("used_range_estimates", false) > - .add("cause", "not available"); > + cause= "not available"; > } The same as above. > } > } > @@ -8217,9 +8215,10 @@ best_access_path(JOIN *join, > best_uses_jbuf= TRUE; > best_filter= 0; > best_type= JT_HASH; > + Json_writer_object trace_access_hash(thd); > trace_access_hash.add("type", "hash"); > trace_access_hash.add("index", "hj-key"); > - trace_access_hash.add("cost", rnd_records); > + trace_access_hash.add("est_records_cost", rnd_records); Please use "rnd_records" instead of "est_records_cost". > trace_access_hash.add("cost", best); > trace_access_hash.add("chosen", true); > } > commit 0e8b82dbcbd8ae96e0758114c87e48061d603c5f > Author: Sergei Krivonos <sergei.krivonos(a)mariadb.com> > Date: Fri Nov 12 03:36:10 2021 +0200 > > MDEV-27036: add assert to detect duplicated JSON keys > > diff --git a/sql/my_json_writer.cc b/sql/my_json_writer.cc > index 9470ba57855..501f0b72318 100644 > --- a/sql/my_json_writer.cc > +++ b/sql/my_json_writer.cc > @@ -39,6 +41,9 @@ inline void Json_writer::on_start_object() > #if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST) > if(!fmt_helper.is_making_writer_calls()) > { > + if(got_name != named_item_expected()){ > + std::cout <<"got_name["<<got_name<<"] != named_item_expected["<<named_item_expected()<<']'<<std::endl; > + } First: as I've already mentioned, please don't write anything to to stderr or stdout directly. Use the sql_print_error() function. Second: the message is not informative. The above will print something like: got_name[1] != named_item_expected[0] which is not at all informative. What's "got_name[1]" ? Please change the printout to be something like: Json_writer: attempt to write invalid JSON. Last data in JSON: %s here %s should print last 256 chars in the JSON document. > VALIDITY_ASSERT(got_name == named_item_expected()); > named_items_expectation.push_back(true); > } > @@ -140,7 +150,19 @@ Json_writer& Json_writer::add_member(const char *name, size_t len) > } > #if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST) > if (!fmt_helper.is_making_writer_calls()) > + { > + VALIDITY_ASSERT(!got_name); > got_name= true; > + VALIDITY_ASSERT(named_items.size()); > + auto& named_items_keys= named_items.top(); > + auto emplaced= named_items_keys.emplace(name, len); > + auto is_uniq_key= emplaced.second; > + if(!is_uniq_key) > + { > + std::cerr << "Duplicated key: " << *emplaced.first << std::endl; > + VALIDITY_ASSERT(is_uniq_key); Same input as above: use sql_print_error and make the error message helpful. > + } > + } > #endif > return *this; > } > commit 57e8f68955954ff96104c8356733bfa73feae849 (HEAD -> bb-10.8-MDEV-27036, origin/bb-10.8-MDEV-27036) > Author: Sergei Krivonos <sergei.krivonos(a)mariadb.com> > Date: Mon Nov 15 05:57:25 2021 +0200 > > MDEV-27036: unittest JSON object member name collision > > diff --git a/unittest/sql/my_json_writer-t.cc b/unittest/sql/my_json_writer-t.cc > index 6398a589c33..7d20802527d 100644 > --- a/unittest/sql/my_json_writer-t.cc > +++ b/unittest/sql/my_json_writer-t.cc > @@ -119,7 +119,14 @@ int main(int args, char **argv) > ok(w.invalid_json, "JSON array end of object"); > } > > - > + { > + Json_writer w; > + w.start_object(); > + w.add_member("name").add_ll(1); > + w.add_member("name").add_ll(2); > + w.end_object(); > + ok(w.invalid_json, "JSON object member name collision"); > + } Please add more unit test coverage. Check that this is accepted: { "foo": { "foo": {} } } Check that this is not accepted: { "foo": { "bar": {} }, "foo": "something else" } BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://petrunia.net

1 0

Re: [Maria-developers] 37b6f5cb8ef: decouple commit/rollback code from binlog hton
by Sergei Golubchik 23 Nov '21

23 Nov '21

Hi, Nikita! On Nov 15, Nikita Malyavin wrote: > revision-id: 37b6f5cb8ef (mariadb-10.5.2-484-g37b6f5cb8ef) > parent(s): f3603cbfaeb > author: Nikita Malyavin > committer: Nikita Malyavin > timestamp: 2021-01-29 03:47:52 +1000 > message: > > decouple commit/rollback code from binlog hton let's squash it with "[6/8] store cache managers in a list" Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 1

Re: [Maria-developers] ad6e7b87107: introduce cache flipping
by Sergei Golubchik 23 Nov '21

23 Nov '21

Hi, Nikita! Interesting trick. On Nov 13, Nikita Malyavin wrote: > revision-id: ad6e7b87107 (mariadb-10.5.2-477-gad6e7b87107) > parent(s): 85fbd867b4c > author: Nikita Malyavin > committer: Nikita Malyavin > timestamp: 2021-01-27 14:17:39 +1000 > message: > > introduce cache flipping Could you add few lines from the Cache_flip_event_log comment here too please? > diff --git a/sql/log.h b/sql/log.h > index 74c409e1ac7..1e422ae4524 100644 > --- a/sql/log.h > +++ b/sql/log.h > @@ -424,10 +424,9 @@ class Event_log: public MYSQL_LOG > > bool open( > const char *log_name, > - enum_log_type log_type, > const char *new_name, ulong next_file_number, > enum cache_type io_cache_type_arg); > - IO_CACHE *get_log_file() { return &log_file; } > + virtual IO_CACHE *get_log_file() { return &log_file; } > > int write_description_event(enum_binlog_checksum_alg checksum_alg, > bool encrypt, bool dont_set_created); > @@ -435,6 +434,73 @@ class Event_log: public MYSQL_LOG > bool write_event(Log_event *ev, binlog_cache_data *data, IO_CACHE *file); > }; > > +/** > + A single-reader, multiple-writer non-blocking layer for Event_log. > + Provides IO_CACHE for writing and IO_CACHE for reading.\ > + > + Writers use an overrided get_log_file version for their writes, while readers > + should use flip() to initiate reading. > + flip() swaps pointers to allow non-blocking reads. > + > + Writers can block other writers and a reader with a mutex, but a reader only > + swaps two pointers under a lock, so it won't block writers. > + > + TODO should be unnecessary after MDEV-24676 is done > + */ > +class Cache_flip_event_log: public Event_log { > + IO_CACHE alt_buf; > + IO_CACHE *current, *alt; > +public: > + > + Cache_flip_event_log() : Event_log(), alt_buf{}, > + current(&log_file), alt(&alt_buf) {} > + bool open(enum cache_type io_cache_type_arg) > + { > + log_file.dir= mysql_tmpdir; > + alt_buf.dir= log_file.dir; > + bool res= Event_log::open(NULL, NULL, 0, io_cache_type_arg); > + if (res) > + return res; > + > + name= my_strdup(key_memory_MYSQL_LOG_name, "online-alter-binlog", > + MYF(MY_WME)); > + if (!name) > + return false; > + > + res= init_io_cache(&alt_buf, -1, LOG_BIN_IO_SIZE, io_cache_type_arg, 0, 0, > + MYF(MY_WME | MY_NABP | MY_WAIT_IF_FULL)) != 0; > + return res; > + } > + > + /** > + Swaps current and alt_log. Can be called only from the reader thread. > + @return a new IO_CACHE pointer to read from. > + */ > + IO_CACHE *flip() > + { > + IO_CACHE *tmp= current; > + reinit_io_cache(alt, WRITE_CACHE, 0, 0, 0); > + mysql_mutex_lock(get_log_lock()); > + reinit_io_cache(current, READ_CACHE, 0, 0, 0); > + current= alt; > + mysql_mutex_unlock(get_log_lock()); > + alt= tmp; > + > + return alt; > + } > + > + IO_CACHE *get_log_file() override > + { > + return current; may be mysql_mutex_assert_owner(get_log_lock()) ? > + } > + > + void cleanup() > + { > + end_io_cache(&alt_buf); > + Event_log::cleanup(); > + } > +}; > + Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 1

Re: [Maria-developers] c80991c79f7: MDEV-25785 Add support for OpenSSL 3.0
by Sergei Golubchik 21 Nov '21

21 Nov '21

Hi, Vladislav! few questions below: On Nov 18, Vladislav Vaintroub wrote: > revision-id: c80991c79f7 (mariadb-10.6.1-213-gc80991c79f7) > parent(s): cee33f1ab7c > author: Vladislav Vaintroub > committer: Vladislav Vaintroub > timestamp: 2021-11-09 02:04:22 +0100 > message: > > MDEV-25785 Add support for OpenSSL 3.0 > > Summary of changes > > - MD_CTX_SIZE is increased > > - EVP_CIPHER_CTX_buf_noconst(ctx) does not work anymore, points > to nobody knows where. The assumption made previously was that > (since the function does not seem to be documented) > was that it points to the last partial source block. > Add own partial block buffer for NOPAD encryption instead > > - SECLEVEL in CipherString in openssl.cnf > had been downgraded to 0, from 1, to make TLSv1.0 and TLSv1.1 possible The definition of SECLEVEL is: Level 0 Everything is permitted. This retains compatibility with previous versions of OpenSSL. Level 1 The security level corresponds to a minimum of 80 bits of security. Any parameters offering below 80 bits of security are excluded. As a result RSA, DSA and DH keys shorter than 1024 bits and ECC keys shorter than 160 bits are prohibited. All export cipher suites are prohibited since they all offer less than 80 bits of security. SSL version 2 is prohibited. Any cipher suite using MD5 for the MAC is also prohibited. Note that signatures using SHA1 and MD5 are also forbidden at this level as they have less than 80 security bits. Only at level 3 does it say "TLS versions below 1.1 are not permitted." I don't see why you had to change from 1 to 0. Do we have "signatures using SHA1 and MD5"? > - ctx_buf buffer now must be aligned to 16 bytes with openssl( > previously with WolfSSL only), ot crashes will happen > > - updated aes-t , to be better debuggable > using function, rather than a huge multiline macro > added test that does "nopad" encryption piece-wise, to test > replacement of EVP_CIPHER_CTX_buf_noconst > > diff --git a/cmake/ssl.cmake b/cmake/ssl.cmake > index 7c2488be8bd..64c93ff9b4f 100644 > --- a/cmake/ssl.cmake > +++ b/cmake/ssl.cmake > @@ -139,9 +139,20 @@ MACRO (MYSQL_CHECK_SSL) > SET(SSL_INTERNAL_INCLUDE_DIRS "") > SET(SSL_DEFINES "-DHAVE_OPENSSL") > > + FOREACH(x INCLUDES LIBRARIES DEFINITIONS) > + SET(SAVE_CMAKE_REQUIRED_${x} ${CMAKE_REQUIRED_${x}}) > + ENDFOREACH() why do you set/restore them? > + # Silence "deprecated in OpenSSL 3.0" > + IF((NOT OPENSSL_VERSION) # 3.0 not determined by older cmake > + OR NOT(OPENSSL_VERSION VERSION_LESS "3.0.0")) > + SET(SSL_DEFINES "${SSL_DEFINES} -DOPENSSL_API_COMPAT=0x10100000L") > + SET(CMAKE_REQUIRED_DEFINITIONS -DOPENSSL_API_COMPAT=0x10100000L) > + ENDIF() This is just postponing the inevitable. They'll drop the old API eventually. As far as I understand the internals were changed in a way that doesn't fit the old API. Is there some safe subset of OpenSSL API that works both in 1.0 and in 3.0 ? It might be more future proof to use only that. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

3 4

Re: [Maria-developers] 4492c869f31: MDEV-26238: Remove inconsistent behaviour of --default-* options in my_print_defaults
by Sergei Golubchik 18 Nov '21

18 Nov '21

Hi, Rucha! Thanks! Looks good now. One comment about tests: On Nov 18, Rucha Deodhar wrote: > revision-id: 4492c869f31 (mariadb-10.6.1-212-g4492c869f31) > parent(s): 3f3ec40c91b > author: Rucha Deodhar > committer: Rucha Deodhar > timestamp: 2021-11-08 16:51:33 +0530 > message: > > MDEV-26238: Remove inconsistent behaviour of --default-* options > in my_print_defaults ... > diff --git a/mysql-test/main/my_print_defaults.test b/mysql-test/main/my_print_defaults.test > index bfd4e563826..b52deaf09ff 100644 > --- a/mysql-test/main/my_print_defaults.test > +++ b/mysql-test/main/my_print_defaults.test > @@ -87,20 +85,74 @@ read_buffer_size=1M ... > +--echo # checking that --defaults* option only works when mentioned at beginning > + > +--echo # Testing --defaults-file at beginning only > +--exec $MYSQL_MY_PRINT_DEFAULTS --defaults-file=$MYSQLTEST_VARDIR/tmp/tmp1.cnf --mysqld > +--remove_file $MYSQLTEST_VARDIR/tmp/tmp3.cnf How does it test that "--defaults* option only works at beginning"? You test that --defaults-file works at the beginning all right. This is redundant, as there are lots of tests in this file that have --defaults-file, but it doesn't hurt. But you don't test the "only" part. You don't test anything that you've changed - I suspect your new test would happily pass without any changes in my_print_defaults.c To test that --defaults-file works *only* at the beginning, you need to verify that it fails not at the beginning. Like error 1 exec $MYSQL_MY_PRINT_DEFAULTS --myslqd --defaults-file=$MYSQLTEST_VARDIR/tmp/tmp1.cnf also with -c error 1 exec $MYSQL_MY_PRINT_DEFAULTS -c $MYSQLTEST_VARDIR/tmp/tmp1.cnf --mysqld This should fail even at the beginning. The point is - you need to test the behavior that you've changed. As your test has introduced failures - you need to test for these failures. But all your tests succeed. > +--echo # Testing --defaults-extra-file works at beginning only > +--exec $MYSQL_MY_PRINT_DEFAULTS --defaults-file=$MYSQLTEST_VARDIR/tmp/tmp1.cnf --defaults-extra-file=$MYSQLTEST_VARDIR/tmp/tmp2.cnf --mysqld Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] MDEV-27036: do not output empty subqueries in traces
by Sergey Petrunia 16 Nov '21

16 Nov '21

Hi Sergei, > commit eb7d6304a2aa168e09d48f6884dadf8c9f868feb > Author: Sergei Krivonos <sergei.krivonos(a)mariadb.com> > Date: Mon Nov 15 01:37:04 2021 +0200 > > do not output empty subqueries in traces This reads as if the trace can have an empty clause describing a subquery. This is very confusing. Actually, the patch removes potential empty "subqueries" arrays. Do you have a testcase for this? I'm trying to find one with grep -rn -A1 '"subqueries"' mysql-test/{main,suite} and I can't. > diff --git a/sql/sql_explain.cc b/sql/sql_explain.cc > index 499b6dbc97a..9a9bf786d62 100644 > --- a/sql/sql_explain.cc > +++ b/sql/sql_explain.cc > @@ -25,6 +25,8 @@ > #include "opt_range.h" > #include "sql_expression_cache.h" > > +#include <algorithm> > + Is this needed? It seems to work without it. > + bool Explain_node::has_json_printable_connection_objects(Explain_query *query) const > + { > + auto n= children.elements(); > + if (n) > + { > + for (; n--;) > + { Why such a convoluted loop inside an if statement? Can you make the code simpler by using for (n=0; n < children.elements(); n++) { if (is_connection_printable_in_json(children.at(i)->connection_type)) return true; } return false; ? > + auto node= query->get_node(children.at(n)); > + /* Derived tables are printed inside Explain_table_access objects */ > + > + if (!is_connection_printable_in_json(node->connection_type)) > + continue; > + > + return true; > + } > + } > + return false; > + } BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://petrunia.net

1 0

Re: [Maria-developers] MDEV-27036: duplicated keys in best_access_path
by Sergey Petrunia 16 Nov '21

16 Nov '21

Hi Sergei, > commit d2b4782200ccfd0850f36d25c75082974e4b86e1 > Author: Sergei Krivonos <sergei.krivonos(a)mariadb.com> > Date: Sun Nov 14 10:37:20 2021 +0200 > > MDEV-27036: duplicated keys in best_access_path > > diff --git a/sql/sql_select.cc b/sql/sql_select.cc > index 3272f8b7f65..29e9c0efe36 100644 > --- a/sql/sql_select.cc > +++ b/sql/sql_select.cc > @@ -7611,16 +7611,14 @@ best_access_path(JOIN *join, > } > else > { > + trace_access_idx.add("used_range_estimates", false); > if (table->quick_keys.is_set(key)) > { > - trace_access_idx.add("used_range_estimates",false) > - .add("cause", > - "not better than ref estimates"); > + cause= "not better than ref estimates"; Issue #1: you make this change here, in the branch marked as ReuseRangeEstimateForRef-2. But there's very similar code right above, in ReuseRangeEstimateForRef-1 branch: /* quick_range couldn't use key! */ records= (double) s->records/rec; trace_access_idx.add("used_range_estimates", false) .add("cause", "not available"); Issue #2: this patch causes some information to be removed from the trace! How about an alternative solution: change the name from "cause" here to "reason"? > } > else > { > - trace_access_idx.add("used_range_estimates", false) > - .add("cause", "not available"); > + cause= "not available"; > } > } > } BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://petrunia.net

1 0

[Maria-developers] MDEV-26929: Please enable optimizer trace run in buildbot JSON
by Sergey Petrunia 09 Nov '21

09 Nov '21

Hi Elena, The patch for https://jira.mariadb.org/browse/MDEV-26929 has been merged from 10.4 into all versions up to 10.8. Could you enable the optimizer trace run in the buildbot please? BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://petrunia.net

2 2

Re: [Maria-developers] 80749146b7d: MDEV-26238: Remove inconsistent behaviour of --default-* options in my_print_defaults
by Sergei Golubchik 07 Nov '21

07 Nov '21

Hi, Rucha! On Nov 05, Rucha Deodhar wrote: > > On Nov 04, Rucha Deodhar wrote: > > > revision-id: 80749146b7d (mariadb-10.6.1-77-g80749146b7d) > > > parent(s): 76149650764 > > > author: Rucha Deodhar > > > committer: Rucha Deodhar > > > timestamp: 2021-09-12 02:11:19 +0530 > > > message: > > > > > > MDEV-26238: Remove inconsistent behaviour of --default-* options > > > in my_print_defaults > > > > > > diff --git a/extra/my_print_defaults.c b/extra/my_print_defaults.c > > > index b7f52382721..607509a309e 100644 > > > --- a/extra/my_print_defaults.c > > > +++ b/extra/my_print_defaults.c > > > @@ -141,50 +124,39 @@ static int get_options(int *argc,char ***argv) > > ... > > > + /* > > > + We already process --defaults* options at the beginning in > > > + get_defaults_options(). So skip --defaults* options and > > > + pass remaining options to handle_options(). > > > + */ > > > + if (is_prefix(org_argv[1], "--no-defaults")) > > > + args_used--; > > > > I'm sorry, I don't understand. Why do you need that? > > I made it to keep the output consistent with previous version. > > If --no-defaults is present in the command line, then it is counted in > args_used when get_defaults_options() is called. Later we skip > "args_used" number of arguments so that only non-default options > mentioned after --default-* options are passed to get_options(). > > The above code decreases this count when --no-defaults is present so > that we don't skip it unlike --defaults-* options and pass it to > get_options(). It is required to pass --no-defaults to get_options() > because if it is not passed then it produces output which looks like > --help ( if no other options are there in addition to --no-defaults). > If it is passed to get_options(), then --no-defaults produces no > output like it does in 10.6. But it's correct, isn't it? If you run my_print_defaults with no groups, it'll print the help text. Try, for example, my_print_defaults --defaults-file=var/my.cnf (presuming you're in mysql-test dir). You'll get the help text, because you didn't specify any groups. Or my_print_defaults --defaults-extra-file=var/my.cnf But my_print_defaults --no-defaults doesn't do it. Doesn't behave like other *defaults* options. We can easily preserve the inconsisent "if --no-defaults, no group is required" behavior with, like if (my_no_defaults) cleanup_and_exit(0); (and add my_no_defaults to my_default.h). But I don't think it's necessary. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 50f43d46299: MDEV-14938 make buildbot to include galera into bintars
by Sergei Golubchik 05 Nov '21

05 Nov '21

Hi, Alexey! On Nov 05, Alexey Bychko wrote: > revision-id: 50f43d46299 (mariadb-10.2.40-106-g50f43d46299) > parent(s): 05c3dced861 > author: Alexey Bychko > committer: Alexey Bychko > timestamp: 2021-10-22 16:30:56 +0700 > message: > > MDEV-14938 make buildbot to include galera into bintars > > this commit adds cmake module to process external directory with built > galera to include galera library and binaries into server build. > it introduces new cmake variable EXT_GALERA_PATH representing relative > or absolute path to compiled galera tree. No compiled galera tree. This is useless. MDEV description says Currently bintars that buildbot produces don't contain libgalera_smm.so. Publishing scripts repackage bintars to add it. We want to produce correct packages in buildbot and release exactly what buildbot builds. It means, buildbot need to produce *exactly* what `prep` script was generating. `prep` script does not build galera. This is a completely separate step, `prep` only packages pre-existing galera files. It's enough to assume that there's ${EXT_GALERA_PATH}/garbd and ${EXT_GALERA_PATH}/libgalera_smm.so. Or if you want it to be really universal, make two different variables, cmake -DBINTAR_GARBD=/path/to/garbd -DBINTAR_LIBGALERA=/path/to/libgalera_smm.so This will work with a compiled galera tree, with installed galera package, with just garbd and libgalera_smm.so in $HOME, with everything. > diff --git a/CMakeLists.txt b/CMakeLists.txt > index 67216e0e443..1107013860c 100644 > --- a/CMakeLists.txt > +++ b/CMakeLists.txt > @@ -153,6 +153,7 @@ INCLUDE(mysql_version) > INCLUDE(cpack_source_ignore_files) > INCLUDE(install_layout) > INCLUDE(wsrep) > +INCLUDE(galera_external) > > # Add macros > INCLUDE(character_sets) > diff --git a/cmake/galera_external.cmake b/cmake/galera_external.cmake > new file mode 100644 > index 00000000000..64ddfc9c498 > --- /dev/null > +++ b/cmake/galera_external.cmake > @@ -0,0 +1,75 @@ > +# EXT_GALERA_PATH > +# a path to built galera git tree > +# or a path to unpacked binary tarball > + > +IF(WIN32 OR NOT EXT_GALERA_PATH) why do you need WIN32 check here? > + RETURN() > +ENDIF() > + > +IF(NOT EXISTS ${EXT_GALERA_PATH}) > + MESSAGE(FATAL_ERROR "Galera dir ${EXT_GALERA_PATH} does not exist!") > +ENDIF() > + > +SET(GALERA_PATH ${CMAKE_SOURCE_DIR}/GALERASYM) > +IF(EXISTS ${GALERA_PATH}) > + FILE({REMOVE ${GALERA_PATH}) > +ENDIF() > +EXECUTE_PROCESS(COMMAND ${CMAKE_COMMAND} -E create_symlink ${EXT_GALERA_PATH} ${GALERA_PATH}) Why do you need this symlink? > + > +FIND_LIBRARY(GALERA_LIB > + NAMES galera_smm galera_enterprise_smm > + PATHS ${GALERA_PATH} > + NO_DEFAULT_PATH > + ) > + > +IF(NOT GALERA_LIB) > + MESSAGE(FATAL_ERROR "Galera library not found in ${GALERA_PATH}") > +ENDIF() > + > +INSTALL(PROGRAMS > + ${GALERA_LIB} > + DESTINATION lib > + RENAME libgalera_smm.so > + COMPONENT Server > + ) > + > +INSTALL(PROGRAMS > + ${GALERA_PATH}/garb/garbd > + DESTINATION bin > + COMPONENT Server > + ) > + > +INSTALL(PROGRAMS > + ${GALERA_PATH}/garb/files/garb-systemd > + DESTINATION bin > + COMPONENT Server > + ) > + > +INSTALL(FILES > + ${GALERA_PATH}/garb/files/garb.cnf > + DESTINATION docs/galera > + COMPONENT Server > + ) MDEV description says Currently bintars that buildbot produces don't contain libgalera_smm.so. Publishing scripts repackage bintars to add it. We want to produce correct packages in buildbot and release exactly what buildbot builds. It means, buildbot need to produce *exactly* what `prep` script was generating. Meaning three galera files. Like: -rwxr-xr-x mariadb-10.2.40-linux-glibc_214-x86_64/bin/garbd -rw-r--r-- mariadb-10.2.40-linux-glibc_214-x86_64/lib/libgalera_smm.so lrwxrwxrwx mariadb-10.2.40-linux-glibc_214-x86_64/lib/galera/libgalera_smm.so -> ../libgalera_smm.so Nothing else. Also, please, verify that the generated directory inside the tarball matches the pattern above. And that the tarbal name does too. > + > +INSTALL(FILES > + ${GALERA_PATH}/AUTHORS > + ${GALERA_PATH}/COPYING > + ${GALERA_PATH}/README > + DESTINATION docs/galera/doc > + COMPONENT Server > + ) > + > +INSTALL(FILES > + ${GALERA_PATH}/asio/LICENSE_1_0.txt > + DESTINATION docs/galera/doc > + RENAME LICENSE.asio > + COMPONENT Server > + ) > + > +INSTALL(FILES > + ${GALERA_PATH}/chromium/LICENSE > + DESTINATION docs/galera/doc > + RENAME LICENSE.chromium > + COMPONENT Server > + ) > + > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] MDEV-23766: Optimizer tracing code is prone to producing invalid JSON
by Sergey Petrunia 05 Nov '21

05 Nov '21

Hi Serg, On Tue, Nov 02, 2021 at 02:19:05AM +0200, Sergei Krivonos wrote: > Applied fixes by your tests patch 638930f85f… <https://github.com/MariaDB/server/commit/638930f85ff40bd39354da81e53e7278a4…> > #1: Please check out the commit adding unit tests: http://lists.askmonty.org/pipermail/commits/2021-November/014773.html It turns out it is not that hard to add them. Any comments/ways to improve this? #2: In the above commit, note that two last unit tests show a scenario that is not currently checked. #3: What is the point of functions Json_writer::on_add_str(), Json_writer::on_start_array, Json_writer::on_start_object() ? I would understand if all the checking code was isolated in these functions. But it isn't. Please either remove the functions or explain the reasoning. #4 See the input below about the commit comment. Please change (feel free to force-push into the branch after that). I think this is all the input, and the patch will be ready for pushing once all of this is addressed. > > Item #2: please improve the commit comment as was requested in my previous > > e-mail: > > > > On Thu, Oct 28, 2021 at 04:58:35PM +0300, Sergey Petrunia wrote: > >> First, input on the commit comment. Please consider it as a request applying > >> to ALL further commits to the MariaDB codebase. > >> > >> > >>> MDEV-23766: implemented requested debug checks > >> > >> Imagine somebody looking at this in a few years. Will they know what checks > >> were requested? They might get a clue by looking at the Jira text, but the > >> idea to make the commit comments concise and self-contained descriptions. > >> > >> Something like this: > >> > >> Line #1: one-line description of the patch: > >> > >>>> MDEV-23766: Make attempts to write invalid JSON cause assertion failures > >> > >> Subsequent lines: > >> > >>>> Make JSON writing API (class Json_writer) check the produced JSON document > >>>> is valid. The following checks are made: > >>>> - JSON objects must contain named members > >>>> - JSON arrays must contain unnamed members. > >>>> - (TODO: add other restrictions we're enforcing). BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://petrunia.net

1 0

Re: [Maria-developers] 80749146b7d: MDEV-26238: Remove inconsistent behaviour of --default-* options in my_print_defaults
by Sergei Golubchik 04 Nov '21

04 Nov '21

Hi, Rucha! On Nov 04, Rucha Deodhar wrote: > revision-id: 80749146b7d (mariadb-10.6.1-77-g80749146b7d) > parent(s): 76149650764 > author: Rucha Deodhar > committer: Rucha Deodhar > timestamp: 2021-09-12 02:11:19 +0530 > message: > > MDEV-26238: Remove inconsistent behaviour of --default-* options > in my_print_defaults > > Analysis: --defaults* option is recognized anywhere in the commandline > instead of only at the beginning because handle_options() recognizes > options in any order. > Fix: use get_defaults_options() which recognizes --defaults* options only at > the beginning. After this is done, we only want to recognize other options > given in any order which can be done using handle_options(). So only skip > --defaults* options and pass rest of them to handle_options(). > Also, removed -e, -g and -c because only my_print_defaults supports them. > > diff --git a/extra/my_print_defaults.c b/extra/my_print_defaults.c > index b7f52382721..607509a309e 100644 > --- a/extra/my_print_defaults.c > +++ b/extra/my_print_defaults.c > @@ -141,50 +124,39 @@ static int get_options(int *argc,char ***argv) ... > int main(int argc, char **argv) > { > - int count= 0, error, no_defaults= 0; > + int count, error, args_used; > char **load_default_groups= 0, *tmp_arguments[6]; > char **argument, **arguments, **org_argv; > int nargs, i= 0; > MY_INIT(argv[0]); > > org_argv= argv; > - if (*argv && !strcmp(*argv, "--no-defaults")) > - { > - argv++; > - ++count; > - no_defaults= 1; > - } > - /* Copy program name and --no-defaults if present*/ > + args_used= get_defaults_options(argv); > + > + /* Copy defaults-xxx arguments & program name */ > + count= args_used; > arguments= tmp_arguments; > - memcpy((char*) arguments, (char*) org_argv, (++count)*sizeof(*org_argv)); > + memcpy((char*) arguments, (char*) org_argv, count*sizeof(*org_argv)); > arguments[count]= 0; I still cannot understand why you're doing it. Why do you need a copy of argv? > > + /* > + We already process --defaults* options at the beginning in > + get_defaults_options(). So skip --defaults* options and > + pass remaining options to handle_options(). > + */ > + if (is_prefix(org_argv[1], "--no-defaults")) > + args_used--; I'm sorry, I don't understand. Why do you need that? all *defaults* options were handled in get_defaults_options(). > + why do you need org_argv? First you do org_argv= argv; then after some code that doesn't change either argv or org_argv, you copy the vallue back: > + argv=org_argv; > + argv+=args_used-1; > + argc-=args_used-1; > + > /* Check out the args */ > if (get_options(&argc,&argv)) > cleanup_and_exit(1); > > - if (!no_defaults) > - { > - if (opt_defaults_file_used) > - arguments[count++]= make_args("--defaults-file=", config_file); > - if (my_defaults_extra_file) > - arguments[count++]= make_args("--defaults-extra-file=", > - my_defaults_extra_file); > - if (my_defaults_group_suffix) > - arguments[count++]= make_args("--defaults-group-suffix=", > - my_defaults_group_suffix); > - arguments[count]= 0; > - } > - > nargs= argc + 1; > if (opt_mysqld) > nargs+= array_elements(mysqld_groups); Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

[Maria-developers] [MDEV-10654] Review request
by Manohar K B 04 Nov '21

04 Nov '21

Hi All, Good day. I have fixed ticket MDEV-10654 and raised PR. Can someone kindly review it? This is the first time I am contributing to MariaDB, so please guide me if I missed anything. Best regards, Manohar.

2 1

Re: [Maria-developers] bb1737b0482: MDEV-12459: The information_schema tables for getting temporary tables info is missing, at least for innodb there is no INNODB_TEMP_TABLE_INFO
by Sergei Golubchik 01 Nov '21

01 Nov '21

Hi, Anel! On Nov 01, Anel Husakovic wrote: > revision-id: bb1737b0482 (mariadb-10.6.1-73-gbb1737b0482) > parent(s): 1bc82aaf0a7 > author: Anel Husakovic > committer: Anel Husakovic > timestamp: 2021-09-06 08:58:46 +0200 > message: > > MDEV-12459: The information_schema tables for getting temporary tables info is missing, at least for innodb there is no INNODB_TEMP_TABLE_INFO > > diff --git a/sql/sql_show.cc b/sql/sql_show.cc > index 6b351726320..5444de8c1ca 100644 > --- a/sql/sql_show.cc > +++ b/sql/sql_show.cc > @@ -69,6 +69,8 @@ > > #include "lex_symbol.h" > #define KEYWORD_SIZE 64 > +#define IS_USER_TEMP_TABLE(A) ((A->tmp_table == TRANSACTIONAL_TMP_TABLE) || \ > + (A->tmp_table == NON_TRANSACTIONAL_TMP_TABLE)) > > extern SYMBOL symbols[]; > extern size_t symbols_length; > @@ -151,6 +153,8 @@ static int show_create_sequence(THD *thd, TABLE_LIST *table_list, > static const LEX_CSTRING *view_algorithm(TABLE_LIST *table); > > bool get_lookup_field_values(THD *, COND *, TABLE_LIST *, LOOKUP_FIELD_VALUES *); > +void process_i_s_table_temporary_tables(THD *thd, TABLE * table, LEX_CSTRING *db_name, > + LEX_CSTRING *table_name, TABLE *tmp_tbl); > > /** > Try to lock a mutex, but give up after a short while to not cause deadlocks > @@ -5161,6 +5166,10 @@ int get_all_tables(THD *thd, TABLE_LIST *tables, COND *cond) > uint table_open_method= tables->table_open_method; > bool can_deadlock; > MEM_ROOT tmp_mem_root; > + Dynamic_array<LEX_CSTRING> system_tables(PSI_INSTRUMENT_MEM); > + All_tmp_tables_list *temp_tables= NULL; > + // Scan for temporary tables > + TMP_TABLE_SHARE *share_temp; > DBUG_ENTER("get_all_tables"); > > bzero(&tmp_mem_root, sizeof(tmp_mem_root)); > @@ -5223,6 +5232,12 @@ int get_all_tables(THD *thd, TABLE_LIST *tables, COND *cond) > init_alloc_root(PSI_INSTRUMENT_ME, &tmp_mem_root, SHOW_ALLOC_BLOCK_SIZE, > SHOW_ALLOC_BLOCK_SIZE, MY_THREAD_SPECIFIC); > > + system_tables.push(INFORMATION_SCHEMA_NAME); > + system_tables.push(PERFORMANCE_SCHEMA_DB_NAME); > + system_tables.push(MYSQL_SCHEMA_NAME); > + system_tables.push(SYS_SCHEMA_NAME); > + system_tables.push(MTR_SCHEMA_NAME); > + > for (size_t i=0; i < db_names.elements(); i++) > { > LEX_CSTRING *db_name= db_names.at(i); > @@ -5234,6 +5249,36 @@ int get_all_tables(THD *thd, TABLE_LIST *tables, COND *cond) > sctx->master_access & (DB_ACLS | SHOW_DB_ACL) || > acl_get(sctx->host, sctx->ip, sctx->priv_user, db_name->str, 0)) > #endif > + > + // Only if there is IS.tables allow temporary tables to be shown > + if (schema_table_idx == SCH_TABLES && !temp_tables) > + { > + for (size_t k=0; k < system_tables.elements(); k++) > + { > + if (db_name != &system_tables.at(k)) > + { > + temp_tables= open_tables_state_backup.temporary_tables; > + if (temp_tables) > + { > + All_tmp_tables_list::Iterator it(*open_tables_state_backup.temporary_tables); > + while ((share_temp= it++)) > + { > + All_share_tables_list::Iterator it2(share_temp->all_tmp_tables); > + while (TABLE *tbl= it2++) > + { > + if (IS_USER_TEMP_TABLE(share_temp)) > + { > + // Now we have the data and we should process_table() manually > + process_i_s_table_temporary_tables(thd, table, db_name, > + &share_temp->table_name, tbl); > + } > + } > + } > + } > + break; > + } > + } > + } hmmmm. What do you do here? I understand, basically, nothing here. Could you explain, line by line? > { > Dynamic_array<LEX_CSTRING*> table_names(PSI_INSTRUMENT_MEM); > int res= make_table_name_list(thd, &table_names, lex, > @@ -5495,6 +5540,8 @@ static int get_schema_tables_record(THD *thd, TABLE_LIST *tables, > table->field[3]->store(STRING_WITH_LEN("SYSTEM VIEW"), cs); > else if (share->table_type == TABLE_TYPE_SEQUENCE) > table->field[3]->store(STRING_WITH_LEN("SEQUENCE"), cs); > + else if (IS_USER_TEMP_TABLE(share)) > + table->field[3]->store(STRING_WITH_LEN("TEMPORARY"), cs); what about INTERNAL_TMP_TABLE and SYSTEM_TMP_TABLE? > else > { > DBUG_ASSERT(share->tmp_table == NO_TMP_TABLE); > @@ -9032,7 +9096,6 @@ ST_FIELD_INFO tables_fields_info[]= > NOT_NULL, "Comment", OPEN_FRM_ONLY), > Column("MAX_INDEX_LENGTH",ULonglong(), NULLABLE, "Max_index_length", > OPEN_FULL_TABLE), > - Column("TEMPORARY", Varchar(1), NULLABLE, "Temporary", OPEN_FRM_ONLY), > CEnd() > }; Let's remove it in a separate commit. After you implement "TEMPORARY" for table type. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] MDEV-23766:Optimizer tracing code is prone to producing invalid JSON
by Sergey Petrunia 31 Oct '21

31 Oct '21

Hi Serg, Ok the patch moving in the right direction but it's not there, yet. Please find the input below. First, input on the commit comment. Please consider it as a request applying to ALL further commits to the MariaDB codebase. > MDEV-23766: implemented requested debug checks Imagine somebody looking at this in a few years. Will they know what checks were requested? They might get a clue by looking at the Jira text, but the idea to make the commit comments concise and self-contained descriptions. Something like this: Line #1: one-line description of the patch: >> MDEV-23766: Make attempts to write invalid JSON cause assertion failures Subsequent lines: >> Make JSON writing API (class Json_writer) check the produced JSON document >> is valid. The following checks are made: >> - JSON objects must contain named members >> - JSON arrays must contain unnamed members. >> - (TODO: add other restrictions we're enforcing). It is not clear how the checker interacts with Single_line_formatting_helper. The code like one the one I'm quoting below shows some non-trivial overlap: > @@ -28,7 +36,16 @@ void Json_writer::append_indent() > > void Json_writer::start_object() > { > +#ifndef NDEBUG > + DBUG_ASSERT(got_name == named_item_expected()); > + named_items_expectation.push_back(true); > + size_t depth= named_items_expectation.size(); > +#endif > fmt_helper.on_start_object(); > +#ifndef NDEBUG > + if (depth != named_items_expectation.size()) > + named_items_expectation.pop_back(); > +#endif > > if (!element_started) > start_element(); This looks very cryptic. Can you elaborate about this? BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://petrunia.net

1 1

Re: [Maria-developers] 85aa3f8cf66: MDEV-26102: dgcov: add support for *.gcda.gcov.json.gz files of gcov 9.1+
by Sergei Golubchik 30 Oct '21

30 Oct '21

Hi, Anel! Looks ok. Except that you have that pointless $gcc_version=~ line that I wrote about in a previous review. Did you verify that this script actually delivers meaningful results? On Oct 30, Anel Husakovic wrote: > revision-id: 85aa3f8cf66 (mariadb-10.2.39-96-g85aa3f8cf66) > parent(s): afe00bb7cce > author: Anel Husakovic > committer: Anel Husakovic > timestamp: 2021-07-29 12:42:44 +0200 > message: > > MDEV-26102: dgcov: add support for *.gcda.gcov.json.gz files of gcov 9.1+ > > Reviewed by: serg(a)mariadb.com > > --- > mysql-test/dgcov.pl | 55 +++++++++++++++++++++++++++++++++++------------------ > 1 file changed, 37 insertions(+), 18 deletions(-) > > diff --git a/mysql-test/dgcov.pl b/mysql-test/dgcov.pl > index 2c00c64d1ff..543099c1b4c 100755 > --- a/mysql-test/dgcov.pl > +++ b/mysql-test/dgcov.pl > @@ -62,12 +62,15 @@ my $res; > + > +my $gcc_version= `gcc -dumpversion`; > +$gcc_version=~ s/(\d).*$/$1/; > > find(\&gcov_one_file, $root); > find(\&write_coverage, $root) if $opt_generate; Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 2c8f52ea53d: MDEV-23328 Server hang due to Galera lock conflict resolution
by Sergei Golubchik 29 Oct '21

29 Oct '21

Hi, Seppo, Jan! this is pretty much the same as in 10.4, so same comments apply, otherwise ok. On Oct 29, sjaakola wrote: > revision-id: 2c8f52ea53d (mariadb-10.5.12-108-g2c8f52ea53d) > parent(s): 910fc72a7f1 > author: sjaakola > committer: Jan Lindström > timestamp: 2021-10-28 12:00:41 +0300 > message: > > MDEV-23328 Server hang due to Galera lock conflict resolution Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 778d88ea2d9: MDEV-23328 Server hang due to Galera lock conflict resolution
by Sergei Golubchik 29 Oct '21

29 Oct '21

Hi, Seppo, Jan! Looks pretty much ok too. A few comments, see below On Oct 29, sjaakola wrote: > revision-id: 778d88ea2d9 (mariadb-10.4.21-76-g778d88ea2d9) > parent(s): e2e4e06f081 > author: sjaakola > committer: Jan Lindström > timestamp: 2021-10-28 11:06:22 +0300 > message: > > MDEV-23328 Server hang due to Galera lock conflict resolution > > diff --git a/sql/log_event.cc b/sql/log_event.cc > index c1c753d69fd..a1562016aa4 100644 > --- a/sql/log_event.cc > +++ b/sql/log_event.cc > @@ -8987,6 +8987,18 @@ bool Xid_log_event::print(FILE* file, PRINT_EVENT_INFO* print_event_info) > > > #if defined(HAVE_REPLICATION) && !defined(MYSQL_CLIENT) > +static bool wsrep_must_replay(THD *thd) > +{ > +#ifdef WITH_WSREP > + mysql_mutex_lock(&thd->LOCK_thd_data); > + bool res= WSREP(thd) && thd->wsrep_trx().state() == wsrep::transaction::s_must_replay; check WSREP(thd) outside of the mutex > + mysql_mutex_unlock(&thd->LOCK_thd_data); > + return res; > +#else > + return false; > +#endif > +} > + > int Xid_log_event::do_apply_event(rpl_group_info *rgi) > { > bool res; > diff --git a/sql/service_wsrep.cc b/sql/service_wsrep.cc > index 6566bb372d8..18067a4d0ec 100644 > --- a/sql/service_wsrep.cc > +++ b/sql/service_wsrep.cc > @@ -29,12 +29,14 @@ extern "C" my_bool wsrep_on(const THD *thd) > > extern "C" void wsrep_thd_LOCK(const THD *thd) > { > + mysql_mutex_lock(&thd->LOCK_thd_kill); > mysql_mutex_lock(&thd->LOCK_thd_data); I presume, you can remove wsrep_thd_kill_LOCK() and wsrep_thd_kill_UNLOCK() that I've added > } > > extern "C" void wsrep_thd_UNLOCK(const THD *thd) > { > mysql_mutex_unlock(&thd->LOCK_thd_data); > + mysql_mutex_unlock(&thd->LOCK_thd_kill); > } > > extern "C" void wsrep_thd_kill_LOCK(const THD *thd) > @@ -295,8 +301,6 @@ extern "C" my_bool wsrep_thd_is_aborting(const MYSQL_THD thd) > default: > return false; > } > - > - return false; this looks like it begs for the compiler warning. please, keep the return where it was. > } Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 002192b92cf: MDEV-23328 Server hang due to Galera lock conflict resolution
by Sergei Golubchik 27 Oct '21

27 Oct '21

Hi, Seppo, Jan! On Oct 27, sjaakola wrote: > revision-id: 002192b92cf (mariadb-10.3.31-60-g002192b92cf) > parent(s): 736da112b65 > author: sjaakola > committer: Jan Lindström > timestamp: 2021-10-27 12:15:48 +0300 > message: > > MDEV-23328 Server hang due to Galera lock conflict resolution This 10.3 version is also ok to push when you have 10.4 and 10.5 versions ready. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] d0e18d694f0: MDEV-23328 Server hang due to Galera lock conflict resolution
by Sergei Golubchik 27 Oct '21

27 Oct '21

Hi, Seppo, Jan! On Oct 27, sjaakola wrote: > revision-id: d0e18d694f0 (mariadb-10.2.40-128-gd0e18d694f0) > parent(s): 408277e8012 > author: sjaakola > committer: Jan Lindström > timestamp: 2021-10-27 12:12:53 +0300 > message: > > MDEV-23328 Server hang due to Galera lock conflict resolution This commit is ok to push, thanks! Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] d1ec0231553: MDEV-26732 Assertion `0' failed in Item::val_native
by Sergei Golubchik 26 Oct '21

26 Oct '21

Hi, Alexander! On Oct 25, Alexander Barkov wrote: > revision-id: d1ec0231553 (mariadb-10.5.4-550-gd1ec0231553) > parent(s): 8b115503563 > author: Alexander Barkov > committer: Alexander Barkov > timestamp: 2021-10-07 20:58:18 +0400 > message: > > MDEV-26732 Assertion `0' failed in Item::val_native > > Also fixes MDEV-24619 Wrong result or Assertion `0' in Item::val_native / Type_handler_inet6::Item_val_native_with_conversion would be good to have some explanation here, what was wrong, what was the fix. > diff --git a/plugin/type_inet/mysql-test/type_inet/type_inet6.result b/plugin/type_inet/mysql-test/type_inet/type_inet6.result > index da949481337..ac16f5c06ce 100644 > --- a/plugin/type_inet/mysql-test/type_inet/type_inet6.result > +++ b/plugin/type_inet/mysql-test/type_inet/type_inet6.result > @@ -2159,3 +2159,38 @@ IFNULL(c, '::1') > ::1 > DROP TABLE t2; > DROP TABLE t1; > +# > +# MDEV-26732 Assertion `0' failed in Item::val_native > +# > +SELECT CAST(CONCAT('::', REPEAT('',RAND())) AS INET6) AS f, var_pop('x') FROM dual HAVING f > ''; do you need RAND() here for a test case? can you replace it with a literal? do you need var_pop() here? > +f var_pop('x') > +Warnings: > +Warning 1292 Truncated incorrect DOUBLE value: 'x' > +Warning 1292 Incorrect inet6 value: '' > +SELECT CAST(CONCAT('::', REPEAT('',RAND())) AS INET6) AS f, var_pop(1) FROM dual HAVING f >= '::'; > +f var_pop(1) > +:: 0.0000 > +CREATE TABLE t1(id INET6 NOT NULL PRIMARY KEY, dsc INET6); > +INSERT INTO t1 VALUES ('::1', '1::1'),('::3', '1::3'),('::4', NULL); > +CREATE TABLE t2 SELECT COALESCE(t1.dsc), COUNT(*) FROM t1 GROUP BY t1.id; > +SELECT * FROM t2 ORDER BY 1,2; > +COALESCE(t1.dsc) COUNT(*) > +NULL 1 > +1::1 1 > +1::3 1 > +DROP TABLE t1, t2; > +# > +# MDEV-24619 Wrong result or Assertion `0' in Item::val_native / Type_handler_inet6::Item_val_native_with_conversion > +# > +CREATE TABLE t1 (a INET6); > +INSERT INTO t1 VALUES ('::'),('::'); > +SELECT IF(1, '::', a) AS f FROM t1 GROUP BY 'foo' HAVING f != ''; > +f > +Warnings: > +Warning 1292 Incorrect inet6 value: '' > +SELECT IF(1, '::', a) AS f FROM t1 GROUP BY 'foo' HAVING f != '::'; > +f > +SELECT IF(1, '::', a) AS f FROM t1 GROUP BY 'foo' HAVING f != '::1'; > +f > +:: > +DROP TABLE t1; Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 5

Re: [Maria-developers] 6d0c1f3ae12: MDEV-23328 Server hang due to Galera lock conflict resolution
by Sergei Golubchik 25 Oct '21

25 Oct '21

Hi, Jan! This was good! Just a few questions, see below On Oct 24, Jan Lindström wrote: > revision-id: 6d0c1f3ae12 (mariadb-10.2.40-114-g6d0c1f3ae12) > parent(s): 3153cc799e2 > author: Jan Lindström > committer: Jan Lindström > timestamp: 2021-10-22 07:02:39 +0300 > message: > > MDEV-23328 Server hang due to Galera lock conflict resolution ... > diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc > index 3e1f248b082..5d40106d0ad 100644 > --- a/sql/sql_parse.cc > +++ b/sql/sql_parse.cc > @@ -2159,6 +2159,7 @@ bool dispatch_command(enum enum_server_command command, THD *thd, > } > DBUG_PRINT("quit",("Got shutdown command for level %u", level)); > general_log_print(thd, command, NullS); > + DBUG_EXECUTE_IF("shutdown_unireg_abort", { unireg_abort(1); }); what's the point of this test? > my_eof(thd); > kill_mysql(thd); > error=TRUE; > diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc > index e60100e2e90..7466654eead 100644 > --- a/sql/wsrep_mysqld.cc > +++ b/sql/wsrep_mysqld.cc > @@ -835,13 +835,25 @@ void wsrep_thr_init() > DBUG_VOID_RETURN; > } > > +/* This is wrapper for wsrep_break_lock in thr_lock.c */ > +static int wsrep_thr_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr, my_bool signal) When is it called? Under what conditions? > +{ > + THD* victim_thd= (THD *) victim_thd_ptr; > + /* We need to lock THD::LOCK_thd_data to protect victim > + from concurrent usage or disconnect or delete. */ > + mysql_mutex_lock(&victim_thd->LOCK_thd_data); > + int res= wsrep_abort_thd(bf_thd_ptr, victim_thd_ptr, signal); > + return res; > +} > + > diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc > index 73b3625b6fd..56400264245 100644 > --- a/storage/innobase/handler/ha_innodb.cc > +++ b/storage/innobase/handler/ha_innodb.cc > @@ -19720,46 +19716,113 @@ wsrep_innobase_kill_one_trx( > +/******************************************************************* > +This function is used to kill one transaction in BF. */ > +void > +wsrep_innobase_kill_one_trx( > + MYSQL_THD const bf_thd, > + const trx_t * const bf_trx, > + trx_t *victim_trx, > + my_bool signal) > +{ > + ut_ad(bf_thd); > + ut_ad(victim_trx); > + ut_ad(lock_mutex_own()); > + ut_ad(trx_mutex_own(victim_trx)); > + > + DBUG_ENTER("wsrep_innobase_kill_one_trx"); > + THD *thd= (THD *) victim_trx->mysql_thd; > + > + /* Here we need to lock THD::LOCK_thd_data to protect from > + concurrent usage or disconnect or delete. */ > + DEBUG_SYNC(bf_thd, "wsrep_before_BF_victim_lock"); > + my_sleep(100000); Eh? Forgot to remove after debugging? > + wsrep_thd_LOCK(thd); > + my_sleep(100000); and here > + DEBUG_SYNC(bf_thd, "wsrep_after_BF_victim_lock"); > + Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

3 5

Re: [Maria-developers] 5c4db10d6ff: MDEV-25114: Crash: WSREP: invalid state ROLLED_BACK (FATAL)
by Sergei Golubchik 24 Oct '21

24 Oct '21

Hi, Jan! On Oct 24, Jan LindstrÃ¶m wrote: > revision-id: 5c4db10d6ff (mariadb-10.3.31-58-g5c4db10d6ff) > parent(s): f9b856b0525 > author: Jan Lindström > committer: Jan Lindström > timestamp: 2021-10-22 07:23:14 +0300 > message: > > MDEV-25114: Crash: WSREP: invalid state ROLLED_BACK (FATAL) > > Revert "MDEV-23328 Server hang due to Galera lock conflict resolution" > > This reverts commit 29bbcac0ee841faaa68eeb09c86ff825eabbe6b6. This was incorrectly reverted. 29bbcac0ee841faaa68eeb09c86ff825eabbe6b6 didn't apply to 10.3 verbatim, there were some merge related changes. To revert it properly you need to look at affected files before the corresponding 10.2->10.3 merge. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] dfa33fb309d: MDEV-26664 Store UUIDs in a more efficient manner
by Sergei Golubchik 22 Oct '21

22 Oct '21

Hi, Alexander! On Oct 19, Alexander Barkov wrote: > revision-id: dfa33fb309d (mariadb-10.6.1-84-gdfa33fb309d) > parent(s): 5cfb31854a4 > author: Alexander Barkov > committer: Alexander Barkov > timestamp: 2021-10-19 13:27:19 +0400 > message: > > MDEV-26664 Store UUIDs in a more efficient manner > > UUID values > > llllllll-mmmm-Vhhh-vsss-nnnnnnnnnnnn > > are now stored as > > nnnnnnnnnnnn-vsss-Vhhh-mmmm-llllllll > > inside the record: > > - the groups (segments separated by dash) are reordered right-to-left. > - the bytes inside the groups are not reordered (stored as before, > in big-endian format). > > This provides a better sorting order: the earlier UUID was generated, > the higher it appears in the ORDER BY output. > > Also, this change enables a good key prefix compression, > because the constant part is now in the beginning, while > the non-constant part (the timestamp) is in the end. > > diff --git a/plugin/type_uuid/mysql-test/type_uuid/type_uuid.result b/plugin/type_uuid/mysql-test/type_uuid/type_uuid.result > index 6a08ffe8c23..6e2e1cc87ef 100644 > --- a/plugin/type_uuid/mysql-test/type_uuid/type_uuid.result > +++ b/plugin/type_uuid/mysql-test/type_uuid/type_uuid.result > @@ -596,10 +596,10 @@ INSERT INTO t1 VALUES (CAST(CONCAT('2','0000000-0000-0000-0000-000000000003') AS > SELECT * FROM t1 ORDER BY a; > a > 10000000-0000-0000-0000-000000000001 > -10000000-0000-0000-0000-000000000002 > -10000000-0000-0000-0000-000000000003 > 20000000-0000-0000-0000-000000000001 > +10000000-0000-0000-0000-000000000002 > 20000000-0000-0000-0000-000000000002 > +10000000-0000-0000-0000-000000000003 > 20000000-0000-0000-0000-000000000003 please, add a test for `ORDER BY CONCAT(a)` just to test the lexicographical ordering of UUIDs. Or may be `ORDER BY HEX(a)` ? Or `UNHEX` ? Whatever the recommended approach should be. > DROP TABLE t1; > # > diff --git a/plugin/type_uuid/mysql-test/type_uuid/type_uuid_memory.result b/plugin/type_uuid/mysql-test/type_uuid/type_uuid_memory.result > index 42bb74d4b01..eb7a51895b9 100644 > --- a/plugin/type_uuid/mysql-test/type_uuid/type_uuid_memory.result > +++ b/plugin/type_uuid/mysql-test/type_uuid/type_uuid_memory.result > @@ -25,7 +25,7 @@ a > 00000000-0000-0000-0000-0000000000ff > EXPLAIN SELECT * FROM t1 WHERE a='00000000-0000-0000-0000-0000000000ff'; > id select_type table type possible_keys key key_len ref rows Extra > -1 SIMPLE t1 ref a a 17 const 2 Using where > +1 SIMPLE t1 ref a a 17 const 4 Using where why? > diff --git a/plugin/type_uuid/sql_type_uuid.h b/plugin/type_uuid/sql_type_uuid.h > index 9076c874e98..be9fea8ebc9 100644 > --- a/plugin/type_uuid/sql_type_uuid.h > +++ b/plugin/type_uuid/sql_type_uuid.h > @@ -24,8 +24,162 @@ class UUID: public FixedBinTypeStorage<MY_UUID_SIZE, MY_UUID_STRING_LENGTH> ... > + // Compare two in-memory values > + static int cmp(const LEX_CSTRING &a, const LEX_CSTRING &b) > + { this is way more LoC that I would've used. but ok, it's your code and it's a plugin, so as you like > diff --git a/sql/sql_string.h b/sql/sql_string.h > index fe57c8153bb..795f80c3e08 100644 > --- a/sql/sql_string.h > +++ b/sql/sql_string.h > @@ -484,6 +484,11 @@ class Binary_string: public Sql_alloc > if (str.Alloced_length) > Alloced_length= (uint32) (str.Alloced_length - offset); > } > + LEX_CSTRING to_lex_cstring() const > + { > + LEX_CSTRING tmp= {Ptr, str_length}; > + return tmp; > + } > inline LEX_CSTRING *get_value(LEX_CSTRING *res) may be you could remove get_value()? In a separate commit. the name is quite bad and the method makes rather little sense, especially if there's to_lex_cstring() now. > { > res->str= Ptr; > diff --git a/sql/sql_type_fixedbin.h b/sql/sql_type_fixedbin.h > index c6e3d20bcfa..5141cb9fad4 100644 > --- a/sql/sql_type_fixedbin.h > +++ b/sql/sql_type_fixedbin.h > @@ -154,18 +160,13 @@ class FixedBinTypeBundle > FbtImpl::max_char_length()+1)); > return false; > } > - int cmp(const char *str, size_t length) const > - { > - DBUG_ASSERT(length == sizeof(m_buffer)); > - return memcmp(m_buffer, str, length); > - } > int cmp(const Binary_string &other) const what is this one used for? > { > - return cmp(other.ptr(), other.length()); > + return FbtImpl::cmp(FbtImpl::to_lex_cstring(), other.to_lex_cstring()); > } > int cmp(const Fbt &other) const > { > - return memcmp(m_buffer, other.m_buffer, sizeof(m_buffer)); > + return FbtImpl::cmp(FbtImpl::to_lex_cstring(), other.to_lex_cstring()); > } > }; > > diff --git a/sql/sql_type_fixedbin_storage.h b/sql/sql_type_fixedbin_storage.h > index 0e8ac81ab59..6e18335bd4c 100644 > --- a/sql/sql_type_fixedbin_storage.h > +++ b/sql/sql_type_fixedbin_storage.h > @@ -21,6 +21,38 @@ > format and their own (variable size) canonical string representation. > > Examples are INET6 and UUID types. > + > + The MariaDB server uses three binary representations of a data type: > + > + 1. In-memory binary representation (user visible) > + This representation: > + - can be used in INSERT..VALUES (X'AABBCC') > + - can be used in WHERE conditions: WHERE c1=X'AABBCC' > + - is returned by CAST(x AS BINARY(N)) > + - is returned by Field::val_native() and Item::val_native() > + > + 2. In-record binary representation (user invisible) > + This representation: > + - is used in records (is pointed by Field::ptr) > + - must be comparable by memcmp() > + > + 3. Binlog binary (row) representation > + Usually, for string data types the binlog representation > + is based on the in-record representation with trailing byte compression: > + - trailing space compression for text string data types > + - trailing zero compression for binary string data types > + > + We have to have separate in-memory and in-record representations > + because we use HA_KEYTYPE_BINARY for indexing. The engine API > + does not have a way to pass a comparison function as a parameter. > + > + The default implementation below assumes that: > + - the in-memory and in-record representations are equal > + - the binlog representation is compatible with BINARY(N) > + This is OK for simple data types, like INET6. > + > + Data type implementations that need different representations > + can override the default implementation (like e.g. UUID does). > */ thanks, this is very good > > /***********************************************************************/ > @@ -64,5 +137,37 @@ class FixedBinTypeStorage > return true; > } > > + static ulong KEY_pack_flags(uint column_nr) > + { > + /* > + Return zero by default. A particular data type can override > + this method return some flags, e.g. HA_PACK_KEY to enable > + key prefix compression. what if you change it later, will it make old tables corrupted? > + */ > + return 0; > + } Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 3

Re: [Maria-developers] 4b164f176e6: MDEV-25114 Crash: WSREP: invalid state ROLLED_BACK (FATAL)
by Sergei Golubchik 21 Oct '21

21 Oct '21

Hi, Seppo, Jan! Note, this is 10.2 patch below. > commit 4b164f176e6 > Author: Seppo Jaakola <seppo.jaakola(a)galeracluster.com> > Date: Wed Sep 15 09:16:44 2021 +0300 > > MDEV-25114 Crash: WSREP: invalid state ROLLED_BACK (FATAL) I think this should say MDEV-23328 Server hang due to Galera lock conflict resolution it'd be easier to understand the commit comment and code changes if it'll refer to the correct problem it is fixing. The fix for MDEV-25114 was the previous commit, the one that This reverts commit 29bbcac0ee841faaa68eeb09c86ff825eabbe6b6 it should have MDEV-25114 in the commit comment. > > This patch is the plan D variant for fixing potetial mutex locking "potential" > order exercised by BF aborting and KILL command execution. > > In this approach, KILL command is replicated as TOI operation. > This guarantees total isolation for the KILL command execution > in the first node: there is no concurrent replication applying > and no concurrent DDL executing. Therefore there is no risk of > BF aborting to happen in parallel with KILL command execution > either. Potential mutex deadlocks between the different mutex > access paths with KILL command execution and BF aborting cannot > therefore happen. > > TOI replication is used, in this approach, purely as means > to provide isolated KILL command execution in the first node. > KILL command should not (and must not) be applied in secondary > nodes. In this patch, we make this sure by skipping KILL > execution in secondary nodes, in applying phase, where we > bail out if applier thread is trying to execute KILL command. > This is effective, but skipping the applying of KILL command > could happen much earlier as well. I like the idea of using TOI apparatus to provide total isolation for KILL. Actually replicating it to nodes is kind of crude, it'd be more elegant not to replicate at all. But it'll do. > > This patch also fixes mutex locking order and unprotected > THD member accesses on bf aborting case. We try to hold > THD::LOCK_thd_data during bf aborting. Only case where it > is not possible is at wsrep_abort_transaction before > call wsrep_innobase_kill_one_trx where we take InnoDB > mutexes first and then THD::LOCK_thd_data. > > This will also fix possible race condition during > close_connection and while wsrep is disconnecting > connections. > > Added wsrep_bf_kill_debug test case > > Reviewed-by: Jan Lindström <jan.lindstrom(a)mariadb.com> > > diff --git a/sql/sql_class.cc b/sql/sql_class.cc > index 5ada018e540..05ec9a1c369 100644 > --- a/sql/sql_class.cc > +++ b/sql/sql_class.cc > @@ -1804,11 +1804,11 @@ void THD::awake(killed_state state_to_set) > the Vio might be disassociated concurrently. > */ > > -void THD::disconnect() > +void THD::disconnect_mutexed() should be disconnect_no_mutex, cf. awake_no_mutex, set_killed_no_mutex. > { > Vio *vio= NULL; > > - mysql_mutex_lock(&LOCK_thd_data); > + mysql_mutex_assert_owner(&LOCK_thd_data); > > set_killed(KILL_CONNECTION); > > diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc > index 3e1f248b082..2bec9c6b6cd 100644 > --- a/sql/sql_parse.cc > +++ b/sql/sql_parse.cc > @@ -9069,6 +9069,18 @@ static > void sql_kill(THD *thd, longlong id, killed_state state, killed_type type) > { > uint error; > +#ifdef WITH_WSREP > + if (WSREP(thd)) > + { > + WSREP_DEBUG("sql_kill called"); > + if (thd->wsrep_applier) > + { > + WSREP_DEBUG("KILL in applying, bailing out here"); > + return; > + } > + WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL) when can it fail and jump to wsrep_error_label? under what conditions? > + } > +#endif /* WITH_WSREP */ > if (!(error= kill_one_thread(thd, id, state, type))) > { > if (!thd->killed) > diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc > index e60100e2e90..00a6dfe2f8a 100644 > --- a/sql/wsrep_mysqld.cc > +++ b/sql/wsrep_mysqld.cc > @@ -835,13 +835,25 @@ void wsrep_thr_init() > DBUG_VOID_RETURN; > } > > +/* This is wrapper for wsrep_break_lock in thr_lock.c */ > +static int wsrep_thr_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr, my_bool signal) > +{ > + THD* victim_thd= (THD *) victim_thd_ptr; > + /* We need to lock THD::LOCK_thd_data to protect victim > + from concurrent usage or disconnect or delete. */ How do you know victim_thd wasn't deleted before you locked LOCK_thd_data below? > + mysql_mutex_lock(&victim_thd->LOCK_thd_data); > + int res= wsrep_abort_thd(bf_thd_ptr, victim_thd_ptr, signal); > + mysql_mutex_unlock(&victim_thd->LOCK_thd_data); > + return res; > +} > > diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc > index 1f666f19d8a..921b8282e45 100644 > --- a/storage/innobase/handler/ha_innodb.cc > +++ b/storage/innobase/handler/ha_innodb.cc > @@ -19739,27 +19739,65 @@ wsrep_abort_transaction( > { > DBUG_ENTER("wsrep_abort_transaction"); > > + ut_ad(bf_thd); > + ut_ad(victim_thd); > trx_t* victim_trx= thd_to_trx(victim_thd); > - trx_t* bf_trx = (bf_thd) ? thd_to_trx(bf_thd) : NULL; > + trx_t* bf_trx= thd_to_trx(bf_thd); > > > if (victim_trx) { > + wsrep_thd_UNLOCK(victim_thd); what keeps victim_trx from disappearing here? > lock_mutex_enter(); > trx_mutex_enter(victim_trx); > wsrep_innobase_kill_one_trx(bf_thd, bf_trx, victim_trx, signal); > lock_mutex_exit(); > trx_mutex_exit(victim_trx); > wsrep_srv_conc_cancel_wait(victim_trx); > + wsrep_thd_LOCK(victim_thd); > DBUG_VOID_RETURN; > } else { > - wsrep_thd_LOCK(victim_thd); > wsrep_thd_set_conflict_state(victim_thd, MUST_ABORT); > - wsrep_thd_UNLOCK(victim_thd); > wsrep_thd_awake(victim_thd, signal); > } Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 20

Re: [Maria-developers] 688ad5ed67f: MDEV-26830: Wrong ROW_NUMBER in diagnostics upon INSERT IGNORE with CHECK violation
by Sergei Golubchik 18 Oct '21

18 Oct '21

Hi, Rucha! On Oct 18, Rucha Deodhar wrote: > revision-id: 688ad5ed67f (mariadb-10.6.1-138-g688ad5ed67f) > parent(s): c27f04ede5a > author: Rucha Deodhar > committer: Rucha Deodhar > timestamp: 2021-10-18 13:40:56 +0530 > message: > > MDEV-26830: Wrong ROW_NUMBER in diagnostics upon INSERT IGNORE with > CHECK violation > > Analysis: When there is constraint fail we return non-zero value for > view_check_option(). So we continue the loop which doesn't increment the > counter because it increments at the end of the loop. > Fix: Increment m_current_row_for_warning() at the beginning of loop. This > will also fix similar bugs if any, about counter not incrementing > correctly because of continue. > > diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc > index 3dff6722a3d..11289779f7c 100644 > --- a/sql/sql_insert.cc > +++ b/sql/sql_insert.cc > @@ -1007,8 +1007,10 @@ bool mysql_insert(THD *thd, TABLE_LIST *table_list, > goto values_loop_end; > } > > + thd->get_stmt_da()->reset_current_row_for_warning(0); This changes the behavior in array binding case, let's not do it in this commit. Better change the previous reset_current_row_for_warning() to reset to 0, not to 1. The rest is ok. > while ((values= its++)) > { > + thd->get_stmt_da()->inc_current_row_for_warning(); > if (fields.elements || !value_count) > { > /* > @@ -1125,7 +1127,6 @@ bool mysql_insert(THD *thd, TABLE_LIST *table_list, > if (unlikely(error)) > break; > info.accepted_rows++; > - thd->get_stmt_da()->inc_current_row_for_warning(); > } > its.rewind(); > iteration++; > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] a895fce3277: MDEV-26842: MDEV-26842: ROW_NUMBER is not set and differs from the message upon WARN_DATA_TRUNCATED produced by inplace ALTER
by Sergei Golubchik 17 Oct '21

17 Oct '21

Hi, Rucha! On Oct 17, Rucha Deodhar wrote: > revision-id: a895fce3277 (mariadb-10.6.1-138-ga895fce3277) > parent(s): c27f04ede5a > author: Rucha Deodhar > committer: Rucha Deodhar > timestamp: 2021-10-17 17:59:11 +0530 > message: > > MDEV-26842: MDEV-26842: ROW_NUMBER is not set and differs from the message > upon WARN_DATA_TRUNCATED produced by inplace ALTER > > Analysis: When row number is passed as parameter to set_warning() it is only > used for error/warning text but m_current_row_for_warning is not updated. > Hence default value of m_current_row_for_warning is assumed. > Fix: update m_current_row_for_warning when error/warning occurs. > > diff --git a/sql/field.cc b/sql/field.cc > index a5cd595f05b..048798ea509 100644 > --- a/sql/field.cc > +++ b/sql/field.cc > @@ -11086,6 +11086,10 @@ Field::set_warning(Sql_condition::enum_warning_level level, uint code, > will have table == NULL. > */ > THD *thd= get_thd(); > + ulong row_count= current_row ? current_row > + : thd->get_stmt_da()->current_row_for_warning(); > + > + thd->get_stmt_da()->reset_current_row_for_warning(row_count); Isn't it a bit confusing, why do you set current_row_for_warning to its own actual value? May be it'd be clearer to if (current_row) thd->get_stmt_da()->reset_current_row_for_warning(current_row); And also, please, mention either here or in the commit comment that it's for INPLACE alter, where the server cannot know what row has generated the warning, so the value of current_row is supplied by the engine. > if (thd->count_cuted_fields > CHECK_FIELD_EXPRESSION) > { > thd->cuted_fields+= cut_increment; > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 351a0a7d6b7: MDEV-26841: ROW_NUMBER is not set and differs from the message upon ER_WRONG_VALUE_COUNT_ON_ROW for the 1st row
by Sergei Golubchik 17 Oct '21

17 Oct '21

Hi, Rucha! On Oct 17, Rucha Deodhar wrote: > revision-id: 351a0a7d6b7 (mariadb-10.6.1-138-g351a0a7d6b7) > parent(s): c27f04ede5a > author: Rucha Deodhar > committer: Rucha Deodhar > timestamp: 2021-10-17 16:24:47 +0530 > message: > > MDEV-26841: ROW_NUMBER is not set and differs from the message upon > ER_WRONG_VALUE_COUNT_ON_ROW for the 1st row > > diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc > index 3dff6722a3d..9e81c33ff8f 100644 > --- a/sql/sql_insert.cc > +++ b/sql/sql_insert.cc > @@ -229,6 +229,8 @@ static int check_insert_fields(THD *thd, TABLE_LIST *table_list, > } > if (values.elements != table->s->visible_fields) > { > + if (values.elements > table->s->visible_fields) why do you need an if() here? > + thd->get_stmt_da()->inc_current_row_for_warning(); > my_error(ER_WRONG_VALUE_COUNT_ON_ROW, MYF(0), 1L); > DBUG_RETURN(-1); > } Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] fcf2c5fff14: MDEV-26299: Some views force server (and mysqldump) to generate invalid SQL for their definitions
by Sergei Golubchik 16 Oct '21

16 Oct '21

Hi, Oleksandr! On Oct 16, Oleksandr Byelkin wrote: > revision-id: fcf2c5fff14 (mariadb-10.2.40-78-gfcf2c5fff14) > parent(s): d5a15f04f4a > author: Oleksandr Byelkin > committer: Oleksandr Byelkin > timestamp: 2021-10-01 14:46:22 +0200 > message: > > MDEV-26299: Some views force server (and mysqldump) to generate invalid SQL for their definitions > > Do not print illegal table field names for non-top-level SELECT list, > they will not be refered in any case but create problem for parsing > of printed result. > > diff --git a/sql/sql_select.cc b/sql/sql_select.cc > index ff584e936b7..ebee9c85791 100644 > --- a/sql/sql_select.cc > +++ b/sql/sql_select.cc > @@ -25797,6 +25797,11 @@ void st_select_lex::print(THD *thd, String *str, enum_query_type query_type) > > //Item List > bool first= 1; > + /* > + outer_select() can not be used here because it is for name resolution > + and will return NULL at any end of name resolution chain (view/derived) > + */ > + bool top_level= (get_master()->get_master() == 0); > List_iterator_fast<Item> it(item_list); > Item *item; > while ((item= it++)) > @@ -25806,7 +25811,8 @@ void st_select_lex::print(THD *thd, String *str, enum_query_type query_type) > else > str->append(','); > > - if (is_subquery_function() && item->is_autogenerated_name) > + if ((is_subquery_function() && item->is_autogenerated_name) || > + !item->name) Where can item->name be NULL? I thought the either specified by the user or autogenerated, but never NULL. > { > /* > Do not print auto-generated aliases in subqueries. It has no purpose > @@ -25815,7 +25821,26 @@ void st_select_lex::print(THD *thd, String *str, enum_query_type query_type) > item->print(str, query_type); > } > else > - item->print_item_w_name(str, query_type); > + { > + /* > + Despite presence of item->name_length it is incorrect in many cases > + till we start using LEX_STRING for the name. > + */ > + size_t len= strlen(item->name); > + > + /* > + Do not print illegal names (if it is not top level SELECT). > + Top level view checked (and correct name are assigned), > + other cases of top level SELECT are not important, because > + it is not "table field". > + */ > + if (top_level || > + !item->is_autogenerated_name || > + !check_table_name(item->name, len, TRUE)) why check_table_name() if it isn't a table? it's an incorrect check to use for columns. > + item->print_item_w_name(str, query_type); > + else > + item->print(str, query_type); > + } > } > > /* > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 2

Re: [Maria-developers] baaaa971662: MDEV-22883: Assertion `!parse_error || lex.sphead == 0' failed in Table_triggers_list::check_n_load | SIGSEGV in strxmov
by Sergei Golubchik 15 Oct '21

15 Oct '21

Hi, Oleksandr! On Oct 15, Oleksandr Byelkin wrote: > revision-id: baaaa971662 (mariadb-10.2.40-61-gbaaaa971662) > parent(s): dce490e9d4d > author: Oleksandr Byelkin > committer: Oleksandr Byelkin > timestamp: 2021-09-19 19:23:49 +0200 > message: > > MDEV-22883: Assertion `!parse_error || lex.sphead == 0' failed in Table_triggers_list::check_n_load | SIGSEGV in strxmov > > Abort execution of SHOW in case of fatal error. Does this patch fixes all test cases from the bug report? There are quite a few. > diff --git a/sql/sql_show.cc b/sql/sql_show.cc > index 721bb053343..105b2719882 100644 > --- a/sql/sql_show.cc > +++ b/sql/sql_show.cc > @@ -4849,7 +4849,8 @@ static int fill_schema_table_from_frm(THD *thd, TABLE *table, > */ > DBUG_ASSERT(thd->open_tables == NULL); > thd->mdl_context.rollback_to_savepoint(open_tables_state_backup->mdl_system_tables_svp); > - thd->clear_error(); > + if (!thd->is_fatal_error) > + thd->clear_error(); Hmm. THD::clear_error() starts with a comment: /** Clear the current error, if any. We do not clear is_fatal_error or is_fatal_sub_stmt_error since we assume this is never called if the fatal error is set. Unfortunately, it doesn't have an assert to go with this, otherwise all these bugs would've probably failed identically and early. Please add DBUG_ASSERT(!is_fatal_error); DBUG_ASSERT(!is_fatal_sub_stmt_error); or something like that to THD::clear_error() > return res; > } > > @@ -5064,7 +5065,7 @@ int get_all_tables(THD *thd, TABLE_LIST *tables, COND *cond) > continue; > } > > - if (thd->killed == ABORT_QUERY) > + if (thd->killed == ABORT_QUERY || thd->is_fatal_error) > { > error= 0; > goto err; that's kind of weird. Why does it continue executing if the statement is killed? Perhaps it'd better be if (thd->killed || thd->is_fatal_error) { if (thd->killed == ABORT_QUERY) error= 0; goto err; Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 2f812e800c0: MDEV-22447: SIGSEGV in Item::val_uint on EXPLAIN
by Sergei Golubchik 15 Oct '21

15 Oct '21

Hi, Oleksandr! On Oct 15, Oleksandr Byelkin wrote: > revision-id: 2f812e800c0 (mariadb-10.2.40-57-g2f812e800c0) > parent(s): 696de6d06c0 > author: Oleksandr Byelkin > committer: Oleksandr Byelkin > timestamp: 2021-09-16 14:12:48 +0200 > message: > > MDEV-22447: SIGSEGV in Item::val_uint on EXPLAIN > > take into account possible NULL > > diff --git a/sql/sql_select.cc b/sql/sql_select.cc > index ff584e936b7..d1b738e94a0 100644 > --- a/sql/sql_select.cc > +++ b/sql/sql_select.cc > @@ -389,7 +389,7 @@ bool handle_select(THD *thd, LEX *lex, select_result *result, > ER_QUERY_EXCEEDED_ROWS_EXAMINED_LIMIT, > ER_THD(thd, ER_QUERY_EXCEEDED_ROWS_EXAMINED_LIMIT), > thd->accessed_rows_and_keys, > - thd->lex->limit_rows_examined->val_uint()); > + thd->lex->get_limit_rows_examined()); > thd->abort_on_warning= saved_abort_on_warning; > thd->reset_killed(); > } How comes, it gets into ER_QUERY_EXCEEDED_ROWS_EXAMINED_LIMIT if limit_rows_examined is NULL? Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] cd4cac2b95a: MDEV-17964: Assertion `status == 0' failed in add_role_user_mapping_action
by Sergei Golubchik 15 Oct '21

15 Oct '21

Hi, Vicențiu! The patch is ok, please push. But given the time it took us to understand the issue, I'm afraid that simply stating that "HASH implementation shuffles elements around" is not enough. Better to explain how elements are shuffled around. For example: in this case the hash has 131 element, DROP ROLE removes the element [119]. Its [119]->next was element [129], so [129] is moved to [119]. Now we need to compact the hash, removing the last element [130]. It gets one bit off its hash value and becomes element [2]. The existing element [2] is moved to [129], and old [130] is moved to [2]. We cannot simply move [130] to [129] and make [2]->next=130, it won't work if [2] is itself in the collision list and doesn't belong in [2]. On Oct 15, Vicențiu Ciorbaru wrote: > revision-id: cd4cac2b95a (mariadb-10.2.40-91-gcd4cac2b95a) > parent(s): 2bb8d7c2f36 > author: Vicențiu Ciorbaru > committer: Vicențiu Ciorbaru > timestamp: 2021-10-13 13:28:50 +0300 > message: > > MDEV-17964: Assertion `status == 0' failed in add_role_user_mapping_action > > This happens upon CREATE USER and DROP ROLE. > > The underlying problem is that our HASH implementation shuffles > elements around when performing an update or delete. This means that > when doing a scan through the HASH table by index, in search of > elements to delete or update one must restart the scan to make sure > nothing is missed if at least one delete / update happened. > > The handle_grant_struct code assumed that it is safe to continue by > only reexamining the currently modified / deleted element index, but > that is not true. > > Missing to delete an element in the hash triggered the assertion in > the test case. DROP ROLE would not clear all necessary role->role or > role->user mappings. > > To fix the problem we ensure that the scan is restarted, only if an > element was deleted / updated, similar to how bubble-sort keeps > sorting until it finds no more elements to swap. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 97458a7767b: MDEV-26832: ROW_NUMBER in SIGNAL/RESIGNAL causes a syntax error
by Sergei Golubchik 15 Oct '21

15 Oct '21

Hi, Rucha! Looks quite good. A couple of small comments, see below. After you fix them, no need to ask for a new review, just push into bb-10.7-row_number On Oct 15, Rucha Deodhar wrote: > revision-id: 97458a7767b (mariadb-10.6.1-131-g97458a7767b) > parent(s): a6cf8b34a83 > author: Rucha Deodhar > committer: Rucha Deodhar > timestamp: 2021-10-15 01:54:51 +0530 > message: > > MDEV-26832: ROW_NUMBER in SIGNAL/RESIGNAL causes a syntax error > > Analysis: Parser was missing ROW_NUMBER as syntax for SIGNAL and RESIGNAL. > Fix: Fix parser and fix how m_row_number is copied like other attributes > to avoid ROW_NUMBER from assuming default value. > > diff --git a/mysql-test/main/get_diagnostics.test b/mysql-test/main/get_diagnostics.test > index 83ea1dee343..186d1bfc438 100644 > --- a/mysql-test/main/get_diagnostics.test > +++ b/mysql-test/main/get_diagnostics.test > @@ -1552,3 +1552,53 @@ GET DIAGNOSTICS CONDITION 2 @rnum = ROW_NUMBER, @msg = MESSAGE_TEXT, @err = MYSQ ... > + > +CREATE PROCEDURE resignal_syntax() > +BEGIN > + DECLARE CONTINUE HANDLER > + FOR 1146 > + BEGIN > + RESIGNAL SET > + MESSAGE_TEXT = '`temptab` does not exist', ROW_NUMBER= 105; > + END; > + SELECT `c` FROM `temptab`; > +END| > + > +DELIMITER ;| > + > +--error ER_NO_SUCH_TABLE > +CALL resignal_syntax(); You test that the above isn't a syntax error. But you also need to verify that row_number was, indeed, set. Like, do GET DIAGNOSTICS and print the ROW_NUMBER. > + > +DROP PROCEDURE resignal_syntax; > diff --git a/sql/sql_error.cc b/sql/sql_error.cc > index a2178831697..84065ddce15 100644 > --- a/sql/sql_error.cc > +++ b/sql/sql_error.cc > @@ -190,6 +190,11 @@ static void copy_string(MEM_ROOT *mem_root, String* dst, const String* src) > dst->length(0); > } > > +void copy_value(ulong *dst, const ulong *src) > +{ > + memcpy(dst, src, sizeof(*src)); > +} > + > void > Sql_condition::copy_opt_attributes(const Sql_condition *cond) > { > @@ -204,6 +209,7 @@ Sql_condition::copy_opt_attributes(const Sql_condition *cond) > copy_string(m_mem_root, & m_table_name, & cond->m_table_name); > copy_string(m_mem_root, & m_column_name, & cond->m_column_name); > copy_string(m_mem_root, & m_cursor_name, & cond->m_cursor_name); > + copy_value(& m_row_number, & cond->m_row_number); That makes no sense. Other values were strings and had to be strdup-ed, but that's a number, you can simply do m_row_number= cond->m_row_number; > } Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] 6ec64caec5d: MDEV-26650: Failed ALTER USER/GRANT statement removes the password from the cache
by Sergei Golubchik 15 Oct '21

15 Oct '21

Hi, Oleksandr! All good, ok to push. Just add a comment right after memcpy, that /* if acl_user->nauth was >= nauth, then we've just updated the actual acl_user, not the copy. We rely on the fact that acl_update_user() cannot fail anywhere below. */ On Oct 15, Oleksandr Byelkin wrote: > revision-id: 6ec64caec5d (mariadb-10.4.21-64-g6ec64caec5d) > parent(s): a736a3174a4 > author: Oleksandr Byelkin > committer: Oleksandr Byelkin > timestamp: 2021-10-14 16:19:09 +0200 > message: > > MDEV-26650: Failed ALTER USER/GRANT statement removes the password from the cache > > Starting from 10.4 AUTH is not part of ACL_USER so changes have to be done > over a copy, and bring in the cache only in case of success. > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] MDEV-26742 Assertion `field->type_handler() == this' failed...
by Sergei Golubchik 12 Oct '21

12 Oct '21

Hi, Alexander! See comments below > diff --git a/plugin/type_inet/mysql-test/type_inet/type_inet6_innodb.test b/plugin/type_inet/mysql-test/type_inet/type_inet6_innodb.test > index dd6049abbf3..3f3a6fdac2c 100644 > --- a/plugin/type_inet/mysql-test/type_inet/type_inet6_innodb.test > +++ b/plugin/type_inet/mysql-test/type_inet/type_inet6_innodb.test > @@ -12,6 +12,30 @@ > SET default_storage_engine=InnoDB; > --source type_inet6_engines.inc > > +--echo # > +--echo # MDEV-26742 Assertion `field->type_handler() == this' failed in FixedBinTypeBundle<NATIVE_LEN, MAX_CHAR_LEN>::Type_handler_fbt::stored_field_cmp_to_item > +--echo # > + > +CREATE TABLE t1 (pk inet6, c text) engine=myisam; > +INSERT INTO t1 VALUES ('::',1); > +CREATE TABLE t2 (d text, KEY (d)) engine=innodb ; > +INSERT INTO t2 VALUES (2); > +--error ER_TRUNCATED_WRONG_VALUE > +UPDATE t2 JOIN t1 ON ( t1.pk > t2.d) SET t1.c = 1; > +SET sql_mode=''; > +UPDATE t2 JOIN t1 ON ( t1.pk > t2.d) SET t1.c = 1; > +SET sql_mode=DEFAULT; > +SELECT * FROM t1; > +SELECT * FROM t2; > +DROP TABLE t1, t2; > + > +CREATE TABLE t1 (pk inet6, c text) engine=myisam; > +INSERT INTO t1 VALUES ('::',1); > +CREATE TABLE t2 ( d text, KEY (d)) engine=innodb ; I don't get it, why do you recreate tables with exactly the same definition and exactly the same content? > +INSERT INTO t2 VALUES (2); > +SELECT * FROM t2 JOIN t1 ON ( t1.pk > t2.d); > +DROP TABLE t1, t2; > + > > --echo # > --echo # End of 10.5 tests > diff --git a/plugin/type_inet/mysql-test/type_inet/type_inet6_myisam.test b/plugin/type_inet/mysql-test/type_inet/type_inet6_myisam.test > index c5183f01cf0..1ccd654181c 100644 > --- a/plugin/type_inet/mysql-test/type_inet/type_inet6_myisam.test > +++ b/plugin/type_inet/mysql-test/type_inet/type_inet6_myisam.test > @@ -10,6 +10,21 @@ > SET default_storage_engine=MyISAM; > --source type_inet6_engines.inc > > +--echo # > +--echo # MDEV-26742 Assertion `field->type_handler() == this' failed in FixedBinTypeBundle<NATIVE_LEN, MAX_CHAR_LEN>::Type_handler_fbt::stored_field_cmp_to_item > +--echo # > + > +CREATE TABLE t1 (c varchar(64), key(c)) engine=myisam; > +INSERT INTO t1 VALUES ('::1'),('::2'); > +SELECT * FROM t1 WHERE c>CAST('::1' AS INET6); > +DROP TABLE t1; > + > +CREATE TABLE t1 (c varchar(64), key(c)) engine=myisam; > +INSERT INTO t1 VALUES ('::1'),('0::1'); also don't see much point in recreating > +SELECT * FROM t1 WHERE c=CAST('::1' AS INET6); > +EXPLAIN SELECT * FROM t1 WHERE c=CAST('::1' AS INET6); > +DROP TABLE t1; > + > > --echo # > --echo # End of 10.5 tests > diff --git a/plugin/type_uuid/mysql-test/type_uuid/type_uuid_innodb.test b/plugin/type_uuid/mysql-test/type_uuid/type_uuid_innodb.test > index 062e25d366b..27e3214af15 100644 > --- a/plugin/type_uuid/mysql-test/type_uuid/type_uuid_innodb.test > +++ b/plugin/type_uuid/mysql-test/type_uuid/type_uuid_innodb.test > @@ -12,5 +12,27 @@ SET default_storage_engine=InnoDB; > --source type_uuid_engines.inc > > --echo # > ---echo # End of 10.5 tests > +--echo # MDEV-26742 Assertion `field->type_handler() == this' failed in FixedBinTypeBundle<NATIVE_LEN, MAX_CHAR_LEN>::Type_handler_fbt::stored_field_cmp_to_item > +--echo # > + > +CREATE TABLE t1 ( pk uuid, c text) engine=myisam; > +INSERT INTO t1 VALUES ('00000000-0000-0000-0000-000000000000',1); > +CREATE TABLE t2 ( d text, KEY (d)) engine=innodb ; > +INSERT INTO t2 VALUES (2); > +--error ER_TRUNCATED_WRONG_VALUE > +UPDATE t2 JOIN t1 ON ( t1.pk > t2.d) SET t1.c = 1; > +SET sql_mode=''; > +UPDATE t2 JOIN t1 ON ( t1.pk > t2.d) SET t1.c = 1; > +SET sql_mode=DEFAULT; > +DROP TABLE t1, t2; > + > +CREATE TABLE t1 (pk uuid, c text) engine=myisam; > +INSERT INTO t1 VALUES ('00000000-0000-0000-0000-000000000000',1); > +CREATE TABLE t2 ( d text, KEY (d)) engine=innodb ; > +INSERT INTO t2 VALUES (2); same here > +SELECT * FROM t2 JOIN t1 ON ( t1.pk > t2.d); > +DROP TABLE t1, t2; > + > +--echo # > +--echo # End of 10.7 tests > --echo # > diff --git a/sql/field.cc b/sql/field.cc > index 46a3a1deea3..fc3456e2ccb 100644 > --- a/sql/field.cc > +++ b/sql/field.cc > @@ -7368,7 +7368,9 @@ bool > Field_longstr::cmp_to_string_with_same_collation(const Item_bool_func *cond, > const Item *item) const > { > + Type_handler_hybrid_field_type cmp(type_handler_for_comparison()); Can this be anything else but type_handler_long_blob ? if not, then I'd suggest Type_handler_hybrid_field_type cmp(type_handler_long_blob); DBUG_ASSERT(type_handler_for_comparison() == &type_handler_long_blob); > + return !cmp.aggregate_for_comparison(item->type_handler_for_comparison()) && > + cmp.type_handler() == type_handler_for_comparison() && > - return item->cmp_type() == STRING_RESULT && > charset() == cond->compare_collation(); This, basically, means, that cmp_type() is now meaningless, one cannot use it anymore? There's no INET6_RESULT and UUID_RESULT for comparison, so what's the point of cmp_type() now? One more question. What type do we have where you need to aggregate here and simple item->type_handler_for_comparison() != &type_handler_long_blob won't do? > } > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 4

[Maria-developers] How can I separate networking test from mysql_test
by Lukas Javorsky 11 Oct '21

11 Oct '21

Hello there, I've open this topic on the maria-discuss mailing list, but, I didn't get any response. Maybe this mailing list could be a better fit for this request. > Hi guys, > I've found few particular tests from the test suite that is failing due to the OpenSSL-3 version and I want to separate them and give them to the OpenSSL team in Red Hat. > > The idea behind this is to give them a set of commands/config options (reproducer) for each test and they will try to find out what's the exact problem there and help us develop the patch. > > The tests that I'm interested in are (there are more of them, I just need to see how to reproduce few of them and I'll do the same for the other ones as well): > encryption.corrupted_during_recovery > encryption.innodb-checksum-algorithm > encryption.innodb-discard-import-change > encryption.innodb_page_encryption_key_change > binlog_encryption.rpl_loaddata_local > binlog_encryption.rpl_parallel_free_deferred_event > > IMPORTANT: These tests are failing only with the 'cbc' cipher in use, the 'ctr' is passing. > > Could you please help me with how to make a reproducer for each test, so I can create them and give them to the OpenSSL team? > > Thank you for any help you can provide > Lukas -- S pozdravom/ Best regards Lukáš Javorský Associate Software Engineer, Core service - Databases Red Hat <https://www.redhat.com> Purkyňova 115 (TPB-C) 612 00 Brno - Královo Pole ljavorsk(a)redhat.com <https://www.redhat.com>

1 0

Re: [Maria-developers] dcbdec60893: MDEV-26650: Failed ALTER USER/GRANT statement removes the password from the cache
by Sergei Golubchik 07 Oct '21

07 Oct '21

Hi, Oleksandr! Ok to push! A couple of typos below On Oct 07, Oleksandr Byelkin wrote: > revision-id: dcbdec60893 (mariadb-10.4.21-56-gdcbdec60893) > parent(s): a75813d4673 > author: Oleksandr Byelkin > committer: Oleksandr Byelkin > timestamp: 2021-10-07 11:39:02 +0200 > message: > > MDEV-26650: Failed ALTER USER/GRANT statement removes the password from the cache > > Starting from 10.4 AUTH is not part of ACL_USER so have to be safely copied. > Patch made in line with preriouse code ideas. "previous" > diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc > index 0e2c044e3a1..cef51038540 100644 > --- a/sql/sql_acl.cc > +++ b/sql/sql_acl.cc > @@ -188,6 +188,9 @@ class ACL_USER_PARAM > class ACL_USER :public ACL_USER_BASE, > public ACL_USER_PARAM > { > +protected: > + // default shellow copy constructor (protect from non-authorized usage) "shallow" > + ACL_USER(const ACL_USER &)= default; > public: Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

[Maria-developers] INSTEAD OF triggers
by Alexandre hadjinlian guerra 05 Oct '21

05 Oct '21

Hello Im looking forward to implement this feature in MariaDB, and will look after trigger definition code/modules in MariaDB, but any wisdom that could be shared to speed up my efforts would be greatly appreciated. The idea is to provide the same functionality as describe here https://aws.amazon.com/pt/blogs/database/implementing-table-partitioning-in… Thanks

3 2

Re: [Maria-developers] 4156071ebe2: MDEV-26606: ERROR_INDEX property value isn't passed from inside a stored procedure
by Sergei Golubchik 28 Sep '21

28 Sep '21

Hi, Rucha! On Sep 28, Rucha Deodhar wrote: > revision-id: 4156071ebe2 (mariadb-10.6.1-91-g4156071ebe2) > parent(s): 8dd4794c4e1 > author: Rucha Deodhar > committer: Rucha Deodhar > timestamp: 2021-09-26 19:58:33 +0530 > message: > > MDEV-26606: ERROR_INDEX property value isn't passed from inside a stored > procedure > > Analysis: m_current_row_for_warning is reset to 1 during cleanup phase of > stored procedure. When we perform a copy because some statement of procedure > created warning, this reset value os passed to push _warning(). > Hence the output is always 1. > Fix: Add a parameter in relevant functions to pass correct value of > error index and don't use m_current_row_for_warning directly. first thought about test cases, ERROR_INDEX should show the same row number as in a warning, so let's check whether warning is correct or not too. In this case we have CREATE OR REPLACE TABLE t1 (pk TINYINT PRIMARY KEY); CREATE OR REPLACE PROCEDURE sp(a INT) INSERT INTO t1 VALUES (2),(a); INSERT INTO t1 VALUES(1); --error ER_WARN_DATA_OUT_OF_RANGE CALL sp(100000); prints ERROR 22003: Out of range value for column 'a' at row 2 which is fine. Would be good to have it in the test. Otherwise looks good. One question below. > diff --git a/sql/sql_class.cc b/sql/sql_class.cc > index 31f5faa72ed..4d7093763ae 100644 > --- a/sql/sql_class.cc > +++ b/sql/sql_class.cc > @@ -1149,7 +1149,8 @@ Sql_condition* THD::raise_condition(uint sql_errno, > if (likely(!(is_fatal_error && (sql_errno == EE_OUTOFMEMORY || > sql_errno == ER_OUTOFMEMORY)))) > { > - cond= da->push_warning(this, sql_errno, sqlstate, level, ucid, msg); > + cond= da->push_warning(this, sql_errno, sqlstate, level, ucid, msg, > + da->current_row_for_warning()); > } > DBUG_RETURN(cond); > } > diff --git a/sql/sql_error.cc b/sql/sql_error.cc > index 7b6e6a83d2f..ca9da3dbf89 100644 > --- a/sql/sql_error.cc > +++ b/sql/sql_error.cc > @@ -663,7 +663,8 @@ void Warning_info::reserve_space(THD *thd, uint count) > > Sql_condition *Warning_info::push_warning(THD *thd, > const Sql_condition_identity *value, > - const char *msg) > + const char *msg, > + ulong current_error_index) > { > Sql_condition *cond= NULL; > > @@ -673,7 +674,7 @@ Sql_condition *Warning_info::push_warning(THD *thd, > m_warn_list.elements() < thd->variables.max_error_count) > { > cond= new (& m_warn_root) Sql_condition(& m_warn_root, *value, msg, > - m_current_row_for_warning); > + current_error_index); > if (cond) > m_warn_list.push_back(cond); > } > @@ -689,7 +690,8 @@ Sql_condition *Warning_info::push_warning(THD *thd, > const Sql_condition *sql_condition) > { > Sql_condition *new_condition= push_warning(thd, sql_condition, > - sql_condition->get_message_text()); > + sql_condition->get_message_text(), > + sql_condition->m_error_index); > > if (new_condition) > new_condition->copy_opt_attributes(sql_condition); > diff --git a/sql/sql_error.h b/sql/sql_error.h > index b6ed7f5061a..a3e00e8419d 100644 > --- a/sql/sql_error.h > +++ b/sql/sql_error.h > @@ -747,7 +747,8 @@ class Warning_info > */ > Sql_condition *push_warning(THD *thd, > const Sql_condition_identity *identity, > - const char* msg); > + const char* msg, > + ulong current_error_index); > > /** > Add a new SQL-condition to the current list and increment the respective > @@ -1178,10 +1179,12 @@ class Diagnostics_area: public Sql_state_errno, > const char* sqlstate, > Sql_condition::enum_warning_level level, > const Sql_user_condition_identity &ucid, > - const char* msg) > + const char* msg, > + ulong current_error_index) > { > Sql_condition_identity tmp(sql_errno_arg, sqlstate, level, ucid); > - return get_warning_info()->push_warning(thd, &tmp, msg); > + return get_warning_info()->push_warning(thd, &tmp, msg, > + current_error_index); > } > > Sql_condition *push_warning(THD *thd, > @@ -1191,7 +1194,7 @@ class Diagnostics_area: public Sql_state_errno, > const char* msg) > { > return push_warning(thd, sqlerrno, sqlstate, level, > - Sql_user_condition_identity(), msg); > + Sql_user_condition_identity(), msg, 0); what is this push_warning used for? does it never need a proper current_error_index? > } > void mark_sql_conditions_for_removal() > { get_warning_info()->mark_sql_conditions_for_removal(); } > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] INSTEAD OF triggers
by Daniel Black 27 Sep '21

27 Sep '21

Opps, meant to send to list too. On Sat, Sep 25, 2021 at 5:28 PM Daniel Black <daniel(a)mariadb.org> wrote: > > Hi Alexandre, > > Given the aws blog is written around that Oracle SE doesn't have > partitions and EE does, does it still make sense with MariaDB (that > includes partitions as standard - > https://mariadb.com/kb/en/partitioning-tables/)? > > Take a look at the way partitions are implemented, they may offer what > you plan to write in a slightly different way. > > If you are sure you want to embark on this journey, there is a > feature request for this issue > https://jira.mariadb.org/browse/MDEV-12151 > > Recommend starting by describing exactly what you plan to implement in > this issue. Reference existing implementations like > https://www.postgresql.org/docs/current/sql-createtrigger.html make > notes of differences and what you plan to cover (and not cover) > > I assume you've come across the generic > https://mariadb.org/get-involved/getting-started-for-developers/ . > Start with a fork from the 10.7 branch > > First bit is sql/sql_yacc.yy trg_action_time that is the parsing syntax. > > If you `git grep TRG_ACTION_AFTER` the code you should contain the > places that will need a TRG_ACTION_INSTEAD_OF handling. > > Also look at the set of tests in $ find mysql-test -name \*trigger\* > > Any problems, find us on > https://mariadb.zulipchat.com/#narrow/stream/118759-general for some > more realtime help. > > On Sat, Sep 25, 2021 at 1:44 AM Alexandre hadjinlian guerra > <alexhguerra(a)gmail.com> wrote: > > > > Hello > > > > Im looking forward to implement this feature in MariaDB, and will look after trigger definition code/modules in MariaDB, but any wisdom that could be shared to speed up my efforts would be greatly appreciated. > > > > The idea is to provide the same functionality as describe here > > https://aws.amazon.com/pt/blogs/database/implementing-table-partitioning-in… > > > > Thanks > > > > _______________________________________________ > > Mailing list: https://launchpad.net/~maria-developers > > Post to : maria-developers(a)lists.launchpad.net > > Unsubscribe : https://launchpad.net/~maria-developers > > More help : https://help.launchpad.net/ListHelp

1 0

Re: [Maria-developers] b248988c8b6: MDEV-19275 Provide SQL service to plugins.
by Sergei Golubchik 14 Sep '21

14 Sep '21

Hi, Alexey! see questions below On Sep 07, Alexey Botchkov wrote: > revision-id: b248988c8b6 (mariadb-10.6.1-74-gb248988c8b6) > parent(s): 42ad4fa3464 > author: Alexey Botchkov > committer: Alexey Botchkov > timestamp: 2021-09-07 14:08:13 +0400 > message: > > MDEV-19275 Provide SQL service to plugins. > > SQL service added. > It provides the limited set of client library functions > to be used by plugin. > > diff --git a/include/mysql/service_sql.h b/include/mysql/service_sql.h > new file mode 100644 > index 00000000000..5050793f655 > --- /dev/null > +++ b/include/mysql/service_sql.h > @@ -0,0 +1,99 @@ > +/* Copyright (C) 2021 MariaDB Corporation > + > + This program is free software; you can redistribute it and/or modify > + it under the terms of the GNU General Public License as published by > + the Free Software Foundation; version 2 of the License. > + > + This program is distributed in the hope that it will be useful, > + but WITHOUT ANY WARRANTY; without even the implied warranty of > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > + GNU General Public License for more details. > + > + You should have received a copy of the GNU General Public License > + along with this program; if not, write to the Free Software > + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ > + > +#if defined(MYSQL_SERVER) && !defined MYSQL_SERVICE_SQL > +#define MYSQL_SERVICE_SQL > + > +#include <mysql.h> > + > +/** > + @file > + SQL service > + > + Interface for plugins to execute SQL queries on the local server. > + > + Functions of the service are the 'server-limited' client library: > + mysql_init > + mysql_real_connect_local > + mysql_real_connect > + mysql_errno > + mysql_error > + mysql_real_query > + mysql_affected_rows > + mysql_num_rows > + mysql_store_result > + mysql_free_result > + mysql_fetch_row > + mysql_close > +*/ > + > + > +#ifdef __cplusplus > +extern "C" { > +#endif > + > +extern struct sql_service_st { > + MYSQL * STDCALL (*mysql_init)(MYSQL *mysql); > + MYSQL *(*mysql_real_connect_local)(MYSQL *mysql, > + const char *host, const char *user, const char *db, > + unsigned long clientflag); > + MYSQL * STDCALL (*mysql_real_connect)(MYSQL *mysql, const char *host, > + const char *user, const char *passwd, const char *db, unsigned int port, > + const char *unix_socket, unsigned long clientflag); > + unsigned int STDCALL (*mysql_errno)(MYSQL *mysql); > + const char * STDCALL (*mysql_error)(MYSQL *mysql); > + int STDCALL (*mysql_real_query)(MYSQL *mysql, const char *q, > + unsigned long length); > + my_ulonglong STDCALL (*mysql_affected_rows)(MYSQL *mysql); > + my_ulonglong STDCALL (*mysql_num_rows)(MYSQL_RES *res); > + MYSQL_RES * STDCALL (*mysql_store_result)(MYSQL *mysql); > + void STDCALL (*mysql_free_result)(MYSQL_RES *result); > + MYSQL_ROW STDCALL (*mysql_fetch_row)(MYSQL_RES *result); > + void STDCALL (*mysql_close)(MYSQL *sock); > +} *sql_service; > + > +#ifdef MYSQL_DYNAMIC_PLUGIN > + > +#define mysql_init sql_service->mysql_init > +#define mysql_real_connect_local sql_service->mysql_real_connect_local > +#define mysql_real_connect sql_service->mysql_real_connect > +#define mysql_errno(M) sql_service->mysql_errno(M) > +#define mysql_error(M) sql_service->mysql_error(M) > +#define mysql_real_query sql_service->mysql_real_query > +#define mysql_affected_rows sql_service->mysql_affected_rows > +#define mysql_num_rows sql_service->mysql_num_rows > +#define mysql_store_result sql_service->mysql_store_result > +#define mysql_free_result sql_service->mysql_free_result > +#define mysql_fetch_row sql_service->mysql_fetch_row > +#define mysql_close sql_service->mysql_close > + > +#else > + > +MYSQL *mysql_real_connect_local(MYSQL *mysql, > + const char *host, const char *user, const char *db, > + unsigned long clientflag); normally the service file is the authoritative source of the service documentation, everything there is to know about the service should be there. In this case it's mostly the normal client C API, so no need to document it. But mysql_real_connect_local is new, so please, add a comment here, explaining what it is. For example, I'd think it connects to the current server, internally. But then, why does it need host and user? The comment should cover it. > + > +/* The rest of the function declarations mest be taken from the mysql.h */ > + > +#endif /*MYSQL_DYNAMIC_PLUGIN*/ > + > + > +#ifdef __cplusplus > +} > +#endif > + > +#endif /*MYSQL_SERVICE_SQL */ > + > + > diff --git a/plugin/test_sql_service/CMakeLists.txt b/plugin/test_sql_service/CMakeLists.txt > index aa9ecfe685e..7c61a1c3c7a 100644 > --- a/plugin/test_sql_service/CMakeLists.txt > +++ b/plugin/test_sql_service/CMakeLists.txt > @@ -15,4 +15,5 @@ > > SET(SOURCES test_sql_service.c) > > -MYSQL_ADD_PLUGIN(test_sql_service ${SOURCES} MODULE_ONLY RECOMPILE_FOR_EMBEDDED) > +ADD_DEFINITIONS(-DMYSQL_SERVER) why? > +MYSQL_ADD_PLUGIN(test_sql_service ${SOURCES} MODULE_ONLY) > diff --git a/plugin/test_sql_service/test_sql_service.c b/plugin/test_sql_service/test_sql_service.c > index 062f10fce58..e1155a98c40 100644 > --- a/plugin/test_sql_service/test_sql_service.c > +++ b/plugin/test_sql_service/test_sql_service.c > @@ -14,71 +14,113 @@ > Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */ > > > -#define PLUGIN_VERSION 0x100 > -#define PLUGIN_STR_VERSION "1.0.0" > - > -#define _my_thread_var loc_thread_var > +#define PLUGIN_VERSION 0x20000 > +#define PLUGIN_STR_VERSION "2.0" > > #include <my_config.h> > -#include <assert.h> > #include <my_global.h> > #include <my_base.h> > -#include <typelib.h> > -//#include <mysql_com.h> /* for enum enum_server_command */ > -#include <mysql/plugin.h> > #include <mysql/plugin_audit.h> > -//#include <string.h> > - > +#include <mysql.h> shouldn't be needed > > -LEX_STRING * thd_query_string (MYSQL_THD thd); > -unsigned long long thd_query_id(const MYSQL_THD thd); > -size_t thd_query_safe(MYSQL_THD thd, char *buf, size_t buflen); > -const char *thd_user_name(MYSQL_THD thd); > -const char *thd_client_host(MYSQL_THD thd); > -const char *thd_client_ip(MYSQL_THD thd); > -LEX_CSTRING *thd_current_db(MYSQL_THD thd); > -int thd_current_status(MYSQL_THD thd); > -enum enum_server_command thd_current_command(MYSQL_THD thd); > - > -int maria_compare_hostname(const char *wild_host, long wild_ip, long ip_mask, > - const char *host, const char *ip); > -void maria_update_hostname(const char **wild_host, long *wild_ip, long *ip_mask, > - const char *host); > > /* Status variables for SHOW STATUS */ > static long test_passed= 0; > +static char *sql_text_local, *sql_text_global; > +static char qwe_res[1024]= ""; > + > static struct st_mysql_show_var test_sql_status[]= > { > {"test_sql_service_passed", (char *)&test_passed, SHOW_LONG}, > + {"test_sql_query_result", qwe_res, SHOW_CHAR}, > {0,0,0} > }; > > static my_bool do_test= TRUE; > -static void run_test(MYSQL_THD thd, struct st_mysql_sys_var *var, > - void *var_ptr, const void *save); > -static MYSQL_SYSVAR_BOOL(run_test, do_test, PLUGIN_VAR_OPCMDARG, > - "Perform the test now.", NULL, run_test, FALSE); > +static int run_test(MYSQL_THD thd, struct st_mysql_sys_var *var, void *save, > + struct st_mysql_value *value); > +static int run_sql_local(MYSQL_THD thd, struct st_mysql_sys_var *var, void *save, > + struct st_mysql_value *value); > +static int run_sql_global(MYSQL_THD thd, struct st_mysql_sys_var *var, void *save, > + struct st_mysql_value *value); > + > +static void noop_update(MYSQL_THD thd, struct st_mysql_sys_var *var, > + void *var_ptr, const void *save); > + > +static MYSQL_SYSVAR_BOOL(run_test, do_test, > + PLUGIN_VAR_OPCMDARG, > + "Perform the test now.", > + run_test, NULL, FALSE); > + > +static MYSQL_SYSVAR_STR(execute_sql_local, sql_text_local, > + PLUGIN_VAR_OPCMDARG, > + "Create the new local connection, execute SQL statement with it.", > + run_sql_local, noop_update, FALSE); > + > +static MYSQL_SYSVAR_STR(execute_sql_global, sql_text_global, > + PLUGIN_VAR_OPCMDARG, > + "Execute SQL statement using the global connection.", > + run_sql_global, noop_update, FALSE); Why did you test local and global connections? What difference in behavior can one expect? > + > static struct st_mysql_sys_var* test_sql_vars[]= > { > MYSQL_SYSVAR(run_test), > + MYSQL_SYSVAR(execute_sql_local), > + MYSQL_SYSVAR(execute_sql_global), > NULL > }; > > diff --git a/sql/mysqld.cc b/sql/mysqld.cc > index ef6f40778fe..8a941a68e53 100644 > --- a/sql/mysqld.cc > +++ b/sql/mysqld.cc > @@ -5064,6 +5075,9 @@ static int init_server_components() > > tc_log= 0; // ha_initialize_handlerton() needs that > > + if (ddl_log_initialize()) > + unireg_abort(1); why? > + > if (plugin_init(&remaining_argc, remaining_argv, > (opt_noacl ? PLUGIN_INIT_SKIP_PLUGIN_TABLE : 0) | > (opt_abort ? PLUGIN_INIT_SKIP_INITIALIZATION : 0))) > diff --git a/sql/sql_class.h b/sql/sql_class.h > index e569fcd32d6..964626be3d4 100644 > --- a/sql/sql_class.h > +++ b/sql/sql_class.h > diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc > index 09ad632dd98..4725855c130 100644 > --- a/sql/sql_prepare.cc > +++ b/sql/sql_prepare.cc > @@ -4857,6 +4870,7 @@ Prepared_statement::execute_server_runnable(Server_runnable *server_runnable) > if (!(lex= new (mem_root) st_lex_local)) > return TRUE; > > + thd->set_time(); why? May be it should be executed using the timestamp of the top-level statement? > thd->set_n_backup_statement(this, &stmt_backup); > thd->set_n_backup_active_arena(this, &stmt_backup); > thd->stmt_arena= this; > @@ -6153,23 +6163,193 @@ static my_bool loc_read_query_result(MYSQL *mysql) > + > extern "C" MYSQL *mysql_real_connect_local(MYSQL *mysql, > - const char *host, const char *user, const char *passwd, const char *db) > + const char *host, const char *user, const char *db, you aren't using `host` or `db` for anything. And I don't understand what you're using `user` for. > + unsigned long clientflag) > { > - //char name_buff[USERNAME_LENGTH]; > - > + THD *thd_orig= current_thd; > + THD *new_thd; > + Protocol_local *p; > DBUG_ENTER("mysql_real_connect_local"); > > /* Test whether we're already connected */ > diff --git a/sql/thread_pool_info.cc b/sql/thread_pool_info.cc > index 90ac6871784..e3ffd160a11 100644 > --- a/sql/thread_pool_info.cc > +++ b/sql/thread_pool_info.cc > @@ -14,9 +14,9 @@ along with this program; if not, write to the Free Software > Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111 - 1301 USA*/ > > #include <mysql_version.h> > -#include <mysql/plugin.h> > > #include <my_global.h> > +#include <mysql/plugin.h> shouldn't be needed, mysql/plugin.h should not need my_global.h > #include <sql_class.h> > #include <sql_i_s.h> > #include <mysql/plugin.h> > Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 6

Re: [Maria-developers] 48261114b9d: MDEV-26572 Improve simple multibyte collation performance on the ASCII range
by Sergei Golubchik 13 Sep '21

13 Sep '21

Hi, Alexander! please add more benchmark results to the MDEV, all the variants that you've tried and speedups (or slowdowns) for each. otherwise, basically, no comments, see below. the only real concern is to know exactly when it helps and when it doesn't. On Sep 11, Alexander Barkov wrote: > revision-id: 48261114b9d (mariadb-10.6.1-77-g48261114b9d) > parent(s): 76149650764 > author: Alexander Barkov > committer: Alexander Barkov > timestamp: 2021-09-09 15:20:57 +0400 > message: > > MDEV-26572 Improve simple multibyte collation performance on the ASCII range > > diff --git a/strings/ctype-ascii.h b/strings/ctype-ascii.h > new file mode 100644 > index 00000000000..66c9cfb3c6d > --- /dev/null > +++ b/strings/ctype-ascii.h > @@ -0,0 +1,189 @@ > +#ifndef CTYPE_ASCII_H_INCLUDED > +#define CTYPE_ASCII_H_INCLUDED The convention is CTYPE_ASCII_INCLUDED (every file has .h, so it's omitted from the include guard) > + > +#include "myisampack.h" > + > +/* > + Magic expression. It uses the fact that for any byte value X in > + the range 0..31 (0x00..0x1F) the expression (X+31)*5 returns > + the 7th bit (0x80) set only for the following six (out of 32) values: > + 0x00, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F. > + These values correspond to offsets of non-letter characters > + in the ASCII table: > + > + The following macro sets the bit 0x20 for the following characters: > + ---------------- -------------------------------- > + Magic bit 10000000000000000000000000011111 > + ASCII 0x00..0x1F ................................ Control > + ASCII 0x20..0x3F ................................ Punctuation, digits > + ASCII 0x40..0x5F @ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_ > + ASCII 0x60..0x7F `abcdefghijklmnopqrstuvwxyz{|}~. > + ---------------- -------------------------------- > + We shift the magic bit 0x80 right twice to make it 0x20. > + So on the ranges [40..5F] and [60..7F] the expression > + has the bit 0x20 set for all non-letter characters. > + Note, other bits contain garbage. > + > + Requirements: > + All bytes must be in the range [00..7F], > + to avoid overflow and carry to the next byte. > +*/ > +#define MY_ASCII_20_IS_SET_IF_NOT_LETTER_MAGIC(i) \ > + (((((i)+0x1F1F1F1F1F1F1F1FULL) & 0x1F1F1F1F1F1F1F1F) * 5) >> 2) > + > + > +/* > + The following macro returns the bit 0x20 set to: > + - 1 for input bytes in the ranges [60..7F] or [E0..FF] > + - 0 otherwise > + Bytes in the ranges [40..7F] and [C0..FF] have the bit 0x40 set. > + Bytes in the ranges [60..7F] smd [E0..FF] have the bit 0x20 set. s/smd/and/ > + Hex BinHi BinLo > + ---- -1-- ---- > + 0x[4C]X .10. .... > + 0x[5D]X .10. .... > + 0x[6E]X .11. .... > + 0x[7F]X .11. .... > +*/ > +#define MY_ASCII_20_IS_SET_IF_RANGE_60_7F_OR_E0_FF(i) (((i) >> 1) & ((i))) > + > + > +/* > + The following macro evaluates to exactly 0x20 for all > + lower case ASCII letters [a-z], and to 0x00 otherwise: > + > + Value Range Character range Subrange > + -------- -------- -------------------------------- ------- > + 00000000 0x00..0x3F Control, punctuation, digits > + 00100000 0x40..0x5F @ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_ letters A-Z > + 00000000 0x40..0x5F @ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_ non-letters > + 00100000 0x60..0x7F `abcdefghijklmnopqrstuvwxyz{|}~. letters a-z > + 00000000 0x60..0x7F `abcdefghijklmnopqrstuvwxyz{|}~. non-letters > + > + Requirements: > + All bytes must be in the range [00..7F]. > + See the comments in MY_ASCII_20_IS_SET_IF_NOT_LETTER_MAGIC(). > +*/ > + > +#define MY_ASCII_20_IF_IS_LOWER_LETTER(i) \ > + (MY_ASCII_20_IS_SET_IF_RANGE_60_7F_OR_E0_FF(i) & \ > + ~MY_ASCII_20_IS_SET_IF_NOT_LETTER_MAGIC(i) & \ > + 0x2020202020202020) > + > +/* > + Convert lower case ASCII letters to upper case by unsetting > + the bit 0x20 with help of the magic expression. > + > + Requirements: > + All bytes must be in the range [00..7F]. > + See the comments in MY_ASCII_20_IS_SET_IF_NOT_LETTER_MAGIC() > +*/ > +#define MY_ASCII_TOUPPER_MAGIC(i) \ > + (i ^ MY_ASCII_20_IF_IS_LOWER_LETTER(i)) > + > + > +/* > + Convert a string (consisting of 8 bytes stored in uint64) > + to upper case algorithmically. > + > + Requirements: > + All bytes must be in the range [00..0x7F]. > + See the comments in MY_ASCII_20_IS_SET_IF_NOT_LETTER_MAGIC(). > + The result on 8bit data is unpredictable!!! > + The caller should make sure not to pass 8bit data. > +*/ > +static inline ulonglong my_ascii_to_upper_magic_uint64(ulonglong i) > +{ > + return MY_ASCII_TOUPPER_MAGIC(i); > +} > + > + > +/* > + Check if: > + - both strings "a" and "b" have at least 4 bytes, and > + - both strings have only 7bit data. > +*/ > +static inline int > +my_strcoll_ascii_4bytes_found(const uchar *a, const uchar *ae, > + const uchar *b, const uchar *be) > +{ > + return a + 4 <= ae && b + 4 <= be && > + (uint4korr(b) & 0x80808080) == 0 && > + (uint4korr(a) & 0x80808080) == 0; > +} > + > + > +/* > + Compare the leading four 7bit ASCII bytes in two strings case insensitively > + by converting letters [a-z] to upper case [A-Z]. > + > + Requirements: > + - The input strings must have at least four bytes, and > + - The leading four bytes in both strings must be 7bit ASCII. > + The caller must make sure to provide only strings that meet > + these requirements. The result on 8-bit data is unpredictable > + as 8-bit bytes may cause overflow in my_ascii_to_upper_magic_uint64(). > + See comments above. > +*/ > +static inline int > +my_strcoll_ascii_toupper_4bytes(const uchar *a, const uchar *b) > +{ > + ulonglong abn= (((ulonglong) mi_uint4korr(a)) << 32) | mi_uint4korr(b); > + abn= my_ascii_to_upper_magic_uint64(abn); > + if ((uint) (abn >> 32) == (uint32) abn) s/uint/uint32/ ? > + return 0; > + return ((uint32) (abn >> 32)) < ((uint32) abn) ? -1 : + 1; > +} > + > + > +/* > + Compare the leading eight 7bit ASCII bytes in two strings case insensitively > + by converting letters [a-z] to upper case [A-Z]. > + > + Requirements: > + - The input strings must have at least eight bytes, and > + - The leading eight bytes in both strings must be 7bit ASCII. > + See comments in my_strcoll_ascii_toupper_4bytes(). > +*/ > +static inline int > +my_strcoll_ascii_toupper_8bytes(const uchar *a, const uchar *b) > +{ > + /* > + TODO: > + Try to get advantage of SIMD instructions by massive comparison > + (16 bytes at a time) of characters against (x>='a' && x<='z') using: > + - either explicit intrinsics I thought your SIMD version was slower, wasn't it? > + - or a loop that can get vectorized automatically by some compilers. > + */ > + ulonglong an= mi_uint8korr(a); > + ulonglong bn= mi_uint8korr(b); > + an= my_ascii_to_upper_magic_uint64(an); > + bn= my_ascii_to_upper_magic_uint64(bn); > + return an == bn ? 0 : an < bn ? -1 : +1; > +} > + > + > +/* > + Compare the leading four 7bit ASCII bytes in two strings in binary style. > +*/ > +static inline int > +my_strcoll_mb7_bin_4bytes(const uchar *a, const uchar *b) > +{ > + uint32 an= mi_uint4korr(a); > + uint32 bn= mi_uint4korr(b); > + return an == bn ? 0 : an < bn ? -1 : +1; > +} > + > + > +/* > + Compare the leading four 7bit ASCII bytes in two strings in binary style. > +*/ > +static inline int > +my_strcoll_mb7_bin_8bytes(const uchar *a, const uchar *b) > +{ > + ulonglong an= mi_uint8korr(a); > + ulonglong bn= mi_uint8korr(b); > + return an == bn ? 0 : an < bn ? -1 : +1; > +} > + > +#endif /* CTYPE_ASCII_H_INCLUDED */ > diff --git a/strings/strcoll.ic b/strings/strcoll.ic > index 6aca0d0c460..72e0131e1c5 100644 > --- a/strings/strcoll.ic > +++ b/strings/strcoll.ic > @@ -39,6 +41,42 @@ > #endif > > > +/* > + For binary collations: > + - on 32bit platforms perform only 4 byte optimization > + - on 64bit platforms perform both 4 byte and 8 byte optimization > +*/ > +#if defined(STRCOLL_MB7_BIN) > +#define MY_STRCOLL_MB7_4BYTES(a,b) my_strcoll_mb7_bin_4bytes((a),(b)) > +#if defined(__x86_64__) as Wlad has already said, what about not gcc-on-intel 64bit platforms? > +#define STRCOLL_MB7_8BYTES > +#define MY_STRCOLL_MB7_8BYTES(a,b) my_strcoll_mb7_bin_8bytes((a),(b)) > +#endif > +#endif > + > + > +/* > + For case insensitive collations with trivial mapping from [a-z] to [A-Z] > + perform optimization only on 64 bit platforms. > + There is no sense to perform my_ascii_to_upper_magic_uint64() based > + optimization on 32bit platforms. The idea of this optimization > + is that it handles 8bytes at a time, using a 64bit CPU register. > + Enabling this optimization on 32bit platform may only slow things down. > +*/ > +#if defined(STRCOLL_MB7_TOUPPER) > +#if defined(__x86_64__) > +#define MY_STRCOLL_MB7_4BYTES(a,b) my_strcoll_ascii_toupper_4bytes((a),(b)) > +#define MY_STRCOLL_MB7_8BYTES(a,b) my_strcoll_ascii_toupper_8bytes((a),(b)) > +#endif /* Architecture test */ > +#endif /* STRCOLL_MB7_TOUPPER */ > + > + > +/* > + A helper macro to shift two pointers forward, to the given amount. > +*/ > +#define MY_STRING_SHIFT_PTR_PTR(a,b,len) do { a+= len; b+= len; } while(0) > + > + > /* > Weight of an illegal byte, must follow these rules: > 1. Must be greater than weight of any normal character in the collation. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

2 1