- developers - lists.mariadb.org

Re: [Maria-developers] 6887f16b61a: MDEV-17399 Add support for JSON_TABLE.
by Sergei Golubchik 06 Feb '20

06 Feb '20

Hi, Alexey! There isn't much to review yet, I don't see how different parts will work together yet. But here you are, some preliminary comments. On Feb 06, Alexey Botchkov wrote: > revision-id: 6887f16b61a (mariadb-10.5.0-153-g6887f16b61a) > parent(s): 74f76206369 > author: Alexey Botchkov <holyfoot(a)mariadb.com> > committer: Alexey Botchkov <holyfoot(a)mariadb.com> > timestamp: 2020-02-04 17:12:55 +0400 > message: > > MDEV-17399 Add support for JSON_TABLE. > > Syntax for the JSON_TABLE added. Standard JSON_TABLE syntax is quite complex. You are doing a subset of it, I presume. It would be good to specify here, in the comment, what syntax exactly are you implementing. > --- > sql/lex.h | 5 ++ > sql/sql_digest.cc | 1 + > sql/sql_lex.h | 3 + > sql/sql_yacc.yy | 164 ++++++++++++++++++++++++++++++++++++++++++++++++++- > sql/table.cc | 1 + > sql/table.h | 2 + > sql/table_function.h | 89 ++++++++++++++++++++++++++++ > 7 files changed, 264 insertions(+), 1 deletion(-) > > diff --git a/sql/lex.h b/sql/lex.h > index f3fc1513369..774f9c1517d 100644 > --- a/sql/lex.h > +++ b/sql/lex.h > @@ -208,6 +208,7 @@ static SYMBOL symbols[] = { > { "ELSE", SYM(ELSE)}, > { "ELSEIF", SYM(ELSEIF_MARIADB_SYM)}, > { "ELSIF", SYM(ELSIF_MARIADB_SYM)}, > + { "EMPTY", SYM(EMPTY_SYM)}, > { "ENABLE", SYM(ENABLE_SYM)}, > { "ENCLOSED", SYM(ENCLOSED)}, > { "END", SYM(END)}, > @@ -414,6 +415,7 @@ static SYMBOL symbols[] = { > { "NATIONAL", SYM(NATIONAL_SYM)}, > { "NATURAL", SYM(NATURAL)}, > { "NCHAR", SYM(NCHAR_SYM)}, > + { "NESTED", SYM(NESTED_SYM)}, > { "NEVER", SYM(NEVER_SYM)}, > { "NEW", SYM(NEW_SYM)}, > { "NEXT", SYM(NEXT_SYM)}, > @@ -448,6 +450,7 @@ static SYMBOL symbols[] = { > { "OPTIONALLY", SYM(OPTIONALLY)}, > { "OR", SYM(OR_SYM)}, > { "ORDER", SYM(ORDER_SYM)}, > + { "ORDINALITY", SYM(ORDINALITY_SYM)}, > { "OTHERS", SYM(OTHERS_MARIADB_SYM)}, > { "OUT", SYM(OUT_SYM)}, > { "OUTER", SYM(OUTER)}, > @@ -460,6 +463,7 @@ static SYMBOL symbols[] = { > { "PAGE_CHECKSUM", SYM(PAGE_CHECKSUM_SYM)}, > { "PARSER", SYM(PARSER_SYM)}, > { "PARSE_VCOL_EXPR", SYM(PARSE_VCOL_EXPR_SYM)}, > + { "PATH", SYM(PATH_SYM)}, > { "PERIOD", SYM(PERIOD_SYM)}, > { "PARTIAL", SYM(PARTIAL)}, > { "PARTITION", SYM(PARTITION_SYM)}, > @@ -742,6 +746,7 @@ static SYMBOL sql_functions[] = { > { "FIRST_VALUE", SYM(FIRST_VALUE_SYM)}, > { "GROUP_CONCAT", SYM(GROUP_CONCAT_SYM)}, > { "JSON_ARRAYAGG", SYM(JSON_ARRAYAGG_SYM)}, > + { "JSON_TABLE", SYM(JSON_TABLE_SYM)}, > { "JSON_OBJECTAGG", SYM(JSON_OBJECTAGG_SYM)}, > { "LAG", SYM(LAG_SYM)}, > { "LEAD", SYM(LEAD_SYM)}, you've added them all as reserved words, is it intentonal? > diff --git a/sql/sql_lex.h b/sql/sql_lex.h > index 308bebd9fa1..591de1e1583 100644 > --- a/sql/sql_lex.h > +++ b/sql/sql_lex.h > @@ -33,6 +33,7 @@ > #include "sql_tvc.h" > #include "item.h" > #include "sql_limit.h" // Select_limit_counters > +#include "table_function.h" // Json_table_column > > /* Used for flags of nesting constructs */ > #define SELECT_NESTING_MAP_SIZE 64 > @@ -3249,6 +3250,8 @@ struct LEX: public Query_tables_list > SQL_I_List<ORDER> proc_list; > SQL_I_List<TABLE_LIST> auxiliary_table_list, save_list; > Column_definition *last_field; > + Json_table_column *cur_json_table_column, *json_table_column_nest; > + Table_function_json_table *json_table; Did you have to put everything in LEX? It's pretty big already and it's never fully used, its members allocate memory all together at the same time, but used for different parts of the grammar, different statements. Could you try to pass that through the parser stack? > Item_sum *in_sum_func; > udf_func udf; > HA_CHECK_OPT check_opt; // check/repair options > diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy > index c00ba42846a..5ef1bbd0721 100644 > --- a/sql/sql_yacc.yy > +++ b/sql/sql_yacc.yy > @@ -207,6 +208,8 @@ void _CONCAT_UNDERSCORED(turn_parser_debug_on,yyparse)() > Lex_for_loop_st for_loop; > Lex_for_loop_bounds_st for_loop_bounds; > Lex_trim_st trim; > + Json_table_column::On_response json_on_response; > + > vers_history_point_t vers_history_point; > struct > { > @@ -225,6 +228,7 @@ void _CONCAT_UNDERSCORED(turn_parser_debug_on,yyparse)() > > /* pointers */ > Create_field *create_field; > + Json_table_column *json_table_column; > Spvar_definition *spvar_definition; > Row_definition_list *spvar_definition_list; > const Type_handler *type_handler; > @@ -308,6 +312,7 @@ void _CONCAT_UNDERSCORED(turn_parser_debug_on,yyparse)() > enum vers_kind_t vers_range_unit; > enum Column_definition::enum_column_versioning vers_column_versioning; > enum plsql_cursor_attr_t plsql_cursor_attr; > + enum Json_table_column::enum_on_type json_on_type; > } > > %{ > @@ -489,6 +494,7 @@ End SQL_MODE_ORACLE_SPECIFIC */ > %token <kwd> ELSEIF_MARIADB_SYM > %token <kwd> ELSE /* SQL-2003-R */ > %token <kwd> ELSIF_ORACLE_SYM /* PLSQL-R */ > +%token <kwd> EMPTY_SYM /* SQL-2016-R */ > %token <kwd> ENCLOSED > %token <kwd> ESCAPED > %token <kwd> EXCEPT_SYM /* SQL-2003-R */ > @@ -506,6 +512,7 @@ End SQL_MODE_ORACLE_SPECIFIC */ > %token <kwd> GRANT /* SQL-2003-R */ > %token <kwd> GROUP_CONCAT_SYM > %token <rwd> JSON_ARRAYAGG_SYM > +%token <rwd> JSON_TABLE_SYM > %token <rwd> JSON_OBJECTAGG_SYM > %token <kwd> GROUP_SYM /* SQL-2003-R */ > %token <kwd> HAVING /* SQL-2003-R */ > @@ -564,6 +571,7 @@ End SQL_MODE_ORACLE_SPECIFIC */ > %token <kwd> MOD_SYM /* SQL-2003-N */ > %token <kwd> NATURAL /* SQL-2003-R */ > %token <kwd> NEG > +%token <kwd> NESTED_SYM /* SQL-2003-N */ No, NESTED is not in the SQL:2003 list of non-reserved words. Please specify a correct comment tag for every keyword you're adding here. > %token <kwd> NOT_SYM /* SQL-2003-R */ > %token <kwd> NO_WRITE_TO_BINLOG > %token <kwd> NOW_SYM > @@ -575,6 +583,7 @@ End SQL_MODE_ORACLE_SPECIFIC */ > %token <kwd> OPTIMIZE > %token <kwd> OPTIONALLY > %token <kwd> ORDER_SYM /* SQL-2003-R */ > +%token <kwd> ORDINALITY_SYM /* SQL-2003-N */ > %token <kwd> OR_SYM /* SQL-2003-R */ > %token <kwd> OTHERS_ORACLE_SYM /* SQL-2011-N, PLSQL-R */ > %token <kwd> OUTER > @@ -585,6 +594,7 @@ End SQL_MODE_ORACLE_SPECIFIC */ > %token <kwd> PAGE_CHECKSUM_SYM > %token <kwd> PARSE_VCOL_EXPR_SYM > %token <kwd> PARTITION_SYM /* SQL-2003-R */ > +%token <kwd> PATH_SYM /* SQL-2003-N */ > %token <kwd> PERCENTILE_CONT_SYM > %token <kwd> PERCENTILE_DISC_SYM > %token <kwd> PERCENT_RANK_SYM > @@ -1336,12 +1346,16 @@ End SQL_MODE_ORACLE_SPECIFIC */ > > %type <type_handler> int_type real_type > > +%type <json_on_type> json_empty_or_error > +%type <json_on_response> json_on_response > + > %type <Lex_field_type> type_with_opt_collate field_type > field_type_numeric > field_type_string > field_type_lob > field_type_temporal > field_type_misc > + json_table_field_type > > %type <Lex_dyncol_type> opt_dyncol_type dyncol_type > numeric_dyncol_type temporal_dyncol_type string_dyncol_type > @@ -1487,7 +1501,7 @@ End SQL_MODE_ORACLE_SPECIFIC */ > table_primary_derived table_primary_derived_opt_parens > derived_table_list table_reference_list_parens > nested_table_reference_list join_table_parens > - update_table_list > + update_table_list table_function > %type <date_time_type> date_time_type; > %type <interval> interval > > @@ -1642,6 +1656,9 @@ End SQL_MODE_ORACLE_SPECIFIC */ > opt_delete_gtid_domain > asrow_attribute > opt_constraint_no_id > + json_table_columns_clause json_table_columns_list json_table_column > + json_table_column_type json_opt_on_empty_or_error > + > > %type <NONE> call sp_proc_stmts sp_proc_stmts1 sp_proc_stmt > %type <NONE> sp_if_then_statements sp_case_then_statements > @@ -11320,6 +11337,150 @@ join_table_list: > derived_table_list { MYSQL_YYABORT_UNLESS($$=$1); } > ; > > +json_table_columns_clause: > + COLUMNS '(' json_table_columns_list ')' > + {} > + ; > + > +json_table_columns_list: > + json_table_column > + | json_table_columns_list ',' json_table_column > + {} > + ; > + > +json_table_column: > + ident > + { > + LEX *lex=Lex; > + Create_field *f= new (thd->mem_root) Create_field(); > + > + if (unlikely(check_string_char_length(&$1, 0, NAME_CHAR_LEN, > + system_charset_info, 1))) > + my_yyabort_error((ER_TOO_LONG_IDENT, MYF(0), $1.str)); > + > + lex->cur_json_table_column= > + new (thd->mem_root) Json_table_column(f, lex->json_table_column_nest); > + > + if (unlikely(!f || !lex->cur_json_table_column)) > + MYSQL_YYABORT; > + > + lex->init_last_field(f, &$1, NULL); > + } > + json_table_column_type > + { > + LEX *lex=Lex; > + lex->json_table->m_columns.push_back(lex->cur_json_table_column, > + thd->mem_root); > + } > + ; > + > +json_table_column_type: > + FOR_SYM ORDINALITY_SYM > + { > + Lex->cur_json_table_column->set(Json_table_column::FOR_ORDINALITY); > + } > + | json_table_field_type PATH_SYM TEXT_STRING_sys > + json_opt_on_empty_or_error > + { > + Lex->last_field->set_attributes(thd, $1, Lex->charset, > + COLUMN_DEFINITION_TABLE_FIELD); > + Lex->cur_json_table_column->set(Json_table_column::PATH, &$3); > + } > + | json_table_field_type EXISTS PATH_SYM TEXT_STRING_sys > + { > + Lex->last_field->set_attributes(thd, $1, Lex->charset, > + COLUMN_DEFINITION_TABLE_FIELD); > + Lex->cur_json_table_column->set(Json_table_column::EXISTS_PATH, &$4); > + } > + | NESTED_SYM PATH_SYM TEXT_STRING_sys > + { > + LEX *lex=Lex; > + lex->cur_json_table_column->set(Json_table_column::NESTED_PATH); > + lex->json_table_column_nest= lex->cur_json_table_column; > + } > + json_table_columns_clause > + { > + LEX *lex=Lex; > + lex->cur_json_table_column= lex->json_table_column_nest; > + lex->json_table_column_nest= lex->cur_json_table_column->m_nest; > + } > + ; > + > +json_table_field_type: > + field_type_numeric > + | field_type_temporal > + | field_type_string > + ; > + > +json_opt_on_empty_or_error: > + /* none */ > + {} > + | json_on_response ON json_empty_or_error json_opt_on_empty_or_error > + { > + if ($3 == Json_table_column::ON_EMPTY) > + { > + Lex->cur_json_table_column->m_on_empty= $1; > + } > + else /* ON_ERROR */ > + { > + Lex->cur_json_table_column->m_on_error= $1; > + } > + } > + ; > + > +json_on_response: > + ERROR_SYM > + { > + $$.m_response= Json_table_column::RESPONSE_ERROR; > + } > + | NULL_SYM > + { > + $$.m_response= Json_table_column::RESPONSE_NULL; > + } > + | DEFAULT TEXT_STRING_sys > + { > + $$.m_response= Json_table_column::RESPONSE_DEFAULT; > + $$.m_default= $1; > + } > + ; > + > +json_empty_or_error: > + ERROR_SYM > + { > + $$= Json_table_column::ON_EMPTY; > + } > + | EMPTY_SYM > + { > + $$= Json_table_column::ON_ERROR; > + } > + ; > + > +table_function: > + JSON_TABLE_SYM '(' expr ',' TEXT_STRING_sys difficult to see what you're parsing without a syntax specs in the commit comment. > + { > + Table_function_json_table *jt= > + new (thd->mem_root) Table_function_json_table($3, &$5); > + if (unlikely(!jt)) > + MYSQL_YYABORT; > + Lex->json_table= jt; > + Lex->json_table_column_nest= NULL; > + } > + json_table_columns_clause ')' AS ident_table_alias > + { > +LEX_CSTRING tn; > + SELECT_LEX *sel= Select; > + sel->table_join_options= 0; > +tn.str= "t1"; > +tn.length= 2; what's that? just to make the code compile for now? why not to use $10 - this is your ident_table_alias, currently it's ignored. > + if (!($$= Select->add_table_to_list(thd, > + new (thd->mem_root) Table_ident(&tn), &tn, > + Select->get_table_join_options(), > + YYPS->m_lock_type, > + YYPS->m_mdl_type))) > + MYSQL_YYABORT; > + } > + ; > + > /* > The ODBC escape syntax for Outer Join is: '{' OJ join_table '}' > The parser does not define OJ as a token, any ident is accepted > @@ -11527,6 +11688,7 @@ table_factor: > $$= $1; > } > | table_reference_list_parens { $$= $1; } > + | table_function { $$= $1; } > ; > > table_primary_ident_opt_parens: Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

[Maria-developers] [DISCUSS] Don't see ARM release information in MariaDB download page
by bo zhaobo 03 Feb '20

03 Feb '20

Hi Team, I want to make clear a thing about the Mariadb download page. Hope team could give some advices or help. As I had sent out an discussion to maria-docs, but didn't get some help. So I think I may get something in dev channel. When I want to download a aarch64 mariadb package, I'm failed to search any information in the content of page [1]. But I tried to search the RPM/DEB package, such as in '*Red Hat, Fedora, and CentOS Packages*' and '*Debian and Ubuntu Packages*', and I can see there are ARM(aarch64/arm64) specific packages in these linked pages [2]. And documentation in [1] just say OS/CPU with "*RedHat/CentOS/Fedora (x86, x86_64, ppc64, ppc64le)*". It seems that there is difference between the documentation with real released packages. That's why I'm here to want to make it clear and discuss with team. *How about add 'aarch64' in the OS/CPU description?* I think that would be great if community could fix that, because it is good for users. They could get what they want from MariaDB download page(MariaDB docs related). Thanks very much. BR ZhaoBo [1] https://downloads.mariadb.org/mariadb/10.5.0/ [2] http://ftp.igh.cnrs.fr/pub/mariadb//mariadb-10.5.0/yum/ http://ftp.igh.cnrs.fr/pub/mariadb//mariadb-10.5.0/repo/ [image: Mailtrack] <https://mailtrack.io?utm_source=gmail&utm_medium=signature&utm_campaign=sig…> Sender notified by Mailtrack <https://mailtrack.io?utm_source=gmail&utm_medium=signature&utm_campaign=sig…> 20/01/06 下午07:31:54

3 7

778e96749bc: MDEV-20923:UBSAN: member access within address … which does not point to an object of type 'xid_count_per_binlog'
by sujatha 23 Jan '20

23 Jan '20

revision-id: 778e96749bcadc5528bb7f214a711272f7413b96 (mariadb-10.1.43-66-g778e96749bc) parent(s): 982294ac1680938ac9223fb64a64e21f0cbc322a author: Sujatha committer: Sujatha timestamp: 2020-01-23 16:17:55 +0530 message: MDEV-20923:UBSAN: member access within address … which does not point to an object of type 'xid_count_per_binlog' Problem: ------- Accessing a member within 'xid_count_per_binlog' structure results in following error when 'UBSAN' is enabled. member access within address 0xXXX which does not point to an object of type 'xid_count_per_binlog' Analysis: --------- The problem appears to be that no constructor for 'xid_count_per_binlog' is being called, and thus the vtable will not be initialized. Fix: --- Defined a parameterized constructor for 'xid_count_per_binlog' class. --- sql/log.cc | 27 +++++++++++---------------- sql/log.h | 13 ++++++++++++- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/sql/log.cc b/sql/log.cc index acf1f8f8a9c..0efef6d1e29 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -3216,7 +3216,7 @@ void MYSQL_BIN_LOG::cleanup() DBUG_ASSERT(!binlog_xid_count_list.head()); WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::cleanup(): Removing xid_list_entry " "for %s (%lu)", b); - my_free(b); + delete b; } mysql_mutex_destroy(&LOCK_log); @@ -3580,17 +3580,9 @@ bool MYSQL_BIN_LOG::open(const char *log_name, */ uint off= dirname_length(log_file_name); uint len= strlen(log_file_name) - off; - char *entry_mem, *name_mem; - if (!(new_xid_list_entry = (xid_count_per_binlog *) - my_multi_malloc(MYF(MY_WME), - &entry_mem, sizeof(xid_count_per_binlog), - &name_mem, len, - NULL))) + new_xid_list_entry= new xid_count_per_binlog(log_file_name+off, (int)len); + if (!new_xid_list_entry) goto err; - memcpy(name_mem, log_file_name+off, len); - new_xid_list_entry->binlog_name= name_mem; - new_xid_list_entry->binlog_name_len= len; - new_xid_list_entry->xid_count= 0; /* Find the name for the Initial binlog checkpoint. @@ -3607,7 +3599,10 @@ bool MYSQL_BIN_LOG::open(const char *log_name, mysql_mutex_unlock(&LOCK_xid_list); if (!b) b= new_xid_list_entry; - strmake(buf, b->binlog_name, b->binlog_name_len); + if (b->binlog_name) + strmake(buf, b->binlog_name, b->binlog_name_len); + else + goto err; Binlog_checkpoint_log_event ev(buf, len); DBUG_EXECUTE_IF("crash_before_write_checkpoint_event", flush_io_cache(&log_file); @@ -3711,7 +3706,7 @@ bool MYSQL_BIN_LOG::open(const char *log_name, { WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::open(): Removing xid_list_entry for " "%s (%lu)", b); - my_free(binlog_xid_count_list.get()); + delete binlog_xid_count_list.get(); } mysql_cond_broadcast(&COND_xid_list); WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::open(): Adding new xid_list_entry for " @@ -3758,7 +3753,7 @@ Turning logging off for the whole duration of the MySQL server process. \ To turn it on again: fix the cause, \ shutdown the MySQL server and restart it.", name, errno); if (new_xid_list_entry) - my_free(new_xid_list_entry); + delete new_xid_list_entry; if (file >= 0) mysql_file_close(file, MYF(0)); close(LOG_CLOSE_INDEX); @@ -4252,7 +4247,7 @@ bool MYSQL_BIN_LOG::reset_logs(THD *thd, bool create_new_log, DBUG_ASSERT(b->xid_count == 0); WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::reset_logs(): Removing " "xid_list_entry for %s (%lu)", b); - my_free(binlog_xid_count_list.get()); + delete binlog_xid_count_list.get(); } mysql_cond_broadcast(&COND_xid_list); reset_master_pending--; @@ -9736,7 +9731,7 @@ TC_LOG_BINLOG::mark_xid_done(ulong binlog_id, bool write_checkpoint) break; WSREP_XID_LIST_ENTRY("TC_LOG_BINLOG::mark_xid_done(): Removing " "xid_list_entry for %s (%lu)", b); - my_free(binlog_xid_count_list.get()); + delete binlog_xid_count_list.get(); } mysql_mutex_unlock(&LOCK_xid_list); diff --git a/sql/log.h b/sql/log.h index b4c9b24a3a9..277e5c6f69c 100644 --- a/sql/log.h +++ b/sql/log.h @@ -587,7 +587,18 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG long xid_count; /* For linking in requests to the binlog background thread. */ xid_count_per_binlog *next_in_queue; - xid_count_per_binlog(); /* Give link error if constructor used. */ + xid_count_per_binlog(char *log_file_name, uint log_file_name_len) + :binlog_id(0), xid_count(0) + { + binlog_name_len= log_file_name_len; + binlog_name= (char *) my_malloc(binlog_name_len, MYF(MY_ZEROFILL)); + if (binlog_name) + memcpy(binlog_name, log_file_name, binlog_name_len); + } + ~xid_count_per_binlog() + { + my_free(binlog_name); + } }; I_List<xid_count_per_binlog> binlog_xid_count_list; mysql_mutex_t LOCK_binlog_background_thread;

1 0

d9716cfadb8: MDEV-20923:UBSAN: member access within address … which does not point to an object of type 'xid_count_per_binlog'
by sujatha 23 Jan '20

23 Jan '20

revision-id: d9716cfadb8cd3448f25d8b411451f87064cecaf (mariadb-10.1.43-66-gd9716cfadb8) parent(s): 982294ac1680938ac9223fb64a64e21f0cbc322a author: Sujatha committer: Sujatha timestamp: 2020-01-23 16:00:41 +0530 message: MDEV-20923:UBSAN: member access within address … which does not point to an object of type 'xid_count_per_binlog' Problem: ------- Accessing a member within 'xid_count_per_binlog' structure results in following error when 'UBSAN' is enabled. member access within address 0xXXX which does not point to an object of type 'xid_count_per_binlog' Analysis: --------- The problem appears to be that no constructor for 'xid_count_per_binlog' is being called, and thus the vtable will not be initialized. Fix: --- Defined a parameterized constructor for 'xid_count_per_binlog' class. --- sql/log.cc | 27 +++++++++++---------------- sql/log.h | 14 +++++++++++++- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/sql/log.cc b/sql/log.cc index acf1f8f8a9c..0efef6d1e29 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -3216,7 +3216,7 @@ void MYSQL_BIN_LOG::cleanup() DBUG_ASSERT(!binlog_xid_count_list.head()); WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::cleanup(): Removing xid_list_entry " "for %s (%lu)", b); - my_free(b); + delete b; } mysql_mutex_destroy(&LOCK_log); @@ -3580,17 +3580,9 @@ bool MYSQL_BIN_LOG::open(const char *log_name, */ uint off= dirname_length(log_file_name); uint len= strlen(log_file_name) - off; - char *entry_mem, *name_mem; - if (!(new_xid_list_entry = (xid_count_per_binlog *) - my_multi_malloc(MYF(MY_WME), - &entry_mem, sizeof(xid_count_per_binlog), - &name_mem, len, - NULL))) + new_xid_list_entry= new xid_count_per_binlog(log_file_name+off, (int)len); + if (!new_xid_list_entry) goto err; - memcpy(name_mem, log_file_name+off, len); - new_xid_list_entry->binlog_name= name_mem; - new_xid_list_entry->binlog_name_len= len; - new_xid_list_entry->xid_count= 0; /* Find the name for the Initial binlog checkpoint. @@ -3607,7 +3599,10 @@ bool MYSQL_BIN_LOG::open(const char *log_name, mysql_mutex_unlock(&LOCK_xid_list); if (!b) b= new_xid_list_entry; - strmake(buf, b->binlog_name, b->binlog_name_len); + if (b->binlog_name) + strmake(buf, b->binlog_name, b->binlog_name_len); + else + goto err; Binlog_checkpoint_log_event ev(buf, len); DBUG_EXECUTE_IF("crash_before_write_checkpoint_event", flush_io_cache(&log_file); @@ -3711,7 +3706,7 @@ bool MYSQL_BIN_LOG::open(const char *log_name, { WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::open(): Removing xid_list_entry for " "%s (%lu)", b); - my_free(binlog_xid_count_list.get()); + delete binlog_xid_count_list.get(); } mysql_cond_broadcast(&COND_xid_list); WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::open(): Adding new xid_list_entry for " @@ -3758,7 +3753,7 @@ Turning logging off for the whole duration of the MySQL server process. \ To turn it on again: fix the cause, \ shutdown the MySQL server and restart it.", name, errno); if (new_xid_list_entry) - my_free(new_xid_list_entry); + delete new_xid_list_entry; if (file >= 0) mysql_file_close(file, MYF(0)); close(LOG_CLOSE_INDEX); @@ -4252,7 +4247,7 @@ bool MYSQL_BIN_LOG::reset_logs(THD *thd, bool create_new_log, DBUG_ASSERT(b->xid_count == 0); WSREP_XID_LIST_ENTRY("MYSQL_BIN_LOG::reset_logs(): Removing " "xid_list_entry for %s (%lu)", b); - my_free(binlog_xid_count_list.get()); + delete binlog_xid_count_list.get(); } mysql_cond_broadcast(&COND_xid_list); reset_master_pending--; @@ -9736,7 +9731,7 @@ TC_LOG_BINLOG::mark_xid_done(ulong binlog_id, bool write_checkpoint) break; WSREP_XID_LIST_ENTRY("TC_LOG_BINLOG::mark_xid_done(): Removing " "xid_list_entry for %s (%lu)", b); - my_free(binlog_xid_count_list.get()); + delete binlog_xid_count_list.get(); } mysql_mutex_unlock(&LOCK_xid_list); diff --git a/sql/log.h b/sql/log.h index b4c9b24a3a9..e69c9c39eaf 100644 --- a/sql/log.h +++ b/sql/log.h @@ -587,7 +587,19 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG long xid_count; /* For linking in requests to the binlog background thread. */ xid_count_per_binlog *next_in_queue; - xid_count_per_binlog(); /* Give link error if constructor used. */ + xid_count_per_binlog(char *log_file_name, uint log_file_name_len) + :binlog_name(log_file_name), binlog_name_len(log_file_name_len), + binlog_id(0), xid_count(0) + { + binlog_name_len= log_file_name_len; + binlog_name= (char *) my_malloc(binlog_name_len, MYF(MY_ZEROFILL)); + if (binlog_name) + memcpy(binlog_name, log_file_name, binlog_name_len); + } + ~xid_count_per_binlog() + { + my_free(binlog_name); + } }; I_List<xid_count_per_binlog> binlog_xid_count_list; mysql_mutex_t LOCK_binlog_background_thread;

1 0

[Maria-developers] 43f27ae4778: MDEV-21490: binlog tests fail with valgrind: Conditional jump or move depends on uninitialised value in sql_ex_info::init
by sujatha 23 Jan '20

23 Jan '20

revision-id: 43f27ae4778b3e7be86897e49c182dc05080fd88 (mariadb-10.1.43-66-g43f27ae4778) parent(s): 982294ac1680938ac9223fb64a64e21f0cbc322a author: Sujatha committer: Sujatha timestamp: 2020-01-23 11:57:46 +0530 message: MDEV-21490: binlog tests fail with valgrind: Conditional jump or move depends on uninitialised value in sql_ex_info::init Problem: ======= P1) Conditional jump or move depends on uninitialised value(s) sql_ex_info::init(char const*, char const*, bool) (log_event.cc:3083) code: All the following variables are not initialized. ---- return ((cached_new_format != -1) ? cached_new_format : (cached_new_format=(field_term_len > 1 || enclosed_len > 1 || line_term_len > 1 || line_start_len > 1 || escaped_len > 1))); P2) Conditional jump or move depends on uninitialised value(s) Rows_log_event::Rows_log_event(char const*, unsigned int, Format_description_log_event const*) (log_event.cc:9571) Code: Uninitialized values is reported for 'var_header_len' variable. ---- if (var_header_len < 2 || event_len < static_cast<unsigned int>(var_header_len + (post_start - buf))) P3) Conditional jump or move depends on uninitialised value(s) Table_map_log_event::pack_info(Protocol*) (log_event.cc:11553) code:'m_table_id' is uninitialized. ---- void Table_map_log_event::pack_info(Protocol *protocol) ... size_t bytes= my_snprintf(buf, sizeof(buf), "table_id: %lu (%s.%s)", m_table_id, m_dbnam, m_tblnam); Fix: === P1 - Fix) Initialize cached_new_format,field_term_len, enclosed_len, line_term_len, line_start_len, escaped_len members in default constructor. P2 - Fix) "var_header_len" is initialized by reading the event buffer. In case of an invalid event the buffer will contain invalid data. Hence added a check to validate the event data. If event_len is smaller than valid header length return immediately. P3 - Fix) 'm_table_id' within Table_map_log_event is initialized by reading data from the event buffer. Use 'VALIDATE_BYTES_READ' macro to validate the current state of the buffer. If it is invalid return immediately. --- sql/log_event.cc | 7 +++++++ sql/log_event.h | 9 ++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/sql/log_event.cc b/sql/log_event.cc index e8881c77f2b..9924b9a0493 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -9535,6 +9535,12 @@ Rows_log_event::Rows_log_event(const char *buf, uint event_len, uint8 const post_header_len= description_event->post_header_len[event_type-1]; + if (event_len < (uint)(common_header_len + post_header_len)) + { + m_cols.bitmap= 0; + DBUG_VOID_RETURN; + } + DBUG_PRINT("enter",("event_len: %u common_header_len: %d " "post_header_len: %d", event_len, common_header_len, @@ -11043,6 +11049,7 @@ Table_map_log_event::Table_map_log_event(const char *buf, uint event_len, const char *post_start= buf + common_header_len; post_start+= TM_MAPID_OFFSET; + VALIDATE_BYTES_READ(post_start, buf, event_len); if (post_header_len == 6) { /* Master is of an intermediate source tree before 5.1.4. Id is 4 bytes */ diff --git a/sql/log_event.h b/sql/log_event.h index 2c8dc3d7353..04fce65faeb 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -2057,7 +2057,14 @@ class Query_log_event: public Log_event ****************************************************************************/ struct sql_ex_info { - sql_ex_info() {} /* Remove gcc warning */ + sql_ex_info(): + cached_new_format(-1), + field_term_len(0), + enclosed_len(0), + line_term_len(0), + line_start_len(0), + escaped_len(0) + {} /* Remove gcc warning */ const char* field_term; const char* enclosed; const char* line_term;

1 0

[Maria-developers] A cleanup for libmariadb
by Alexander Barkov 20 Jan '20

20 Jan '20

Hi Georg, Can you please review this cleanup patch? It's a prerequisite for: MDEV-17832 Protocol: extensions for Pluggable types and JSON, GEOMETRY The idea is that this patch introduces a new file libmariadb/mariadb_priv.h and adds prototypes of these functions: void free_rows(MYSQL_DATA *cur); int ma_multi_command(MYSQL *mysql, enum enum_multi_status status); MYSQL_FIELD * unpack_fields(MYSQL_DATA *data, MA_MEM_ROOT *alloc,uint fields, my_bool default_value); and removes these prototypes from mariadb_stmt.c. I think it should be safer this way. Later I'll add there more prototypes into libmariadb/mariadb_priv.h, e.g. functions that will be added under terms of MDEV-17832. Note, I also removed the "my_bool long_flag_protocol" parameter from unpack_fields(), it's not used anyway. Under terms of MDEV-17832 I'll have to add a "const MYSQL *mysql" parameter. So if we ever need to check CLIENT_LONG_FLAG inside unpack_fields() again, it can be done through the "mysql" parameter: (mysql->server_capabilities & CLIENT_LONG_FLAG) Thanks!

1 0

[Maria-developers] A cleanup for libmariadb
by Alexander Barkov 20 Jan '20

20 Jan '20

Hi Georg, Can you please review this cleanup patch? It's a prerequisite for: MDEV-17832 Protocol: extensions for Pluggable types and JSON, GEOMETRY The idea is that this patch introduces a new file libmariadb/mariadb_priv.h and adds prototypes of these functions: void free_rows(MYSQL_DATA *cur); int ma_multi_command(MYSQL *mysql, enum enum_multi_status status); MYSQL_FIELD * unpack_fields(MYSQL_DATA *data, MA_MEM_ROOT *alloc,uint fields, my_bool default_value); and removes these prototypes from mariadb_stmt.c. I think it should be safer this way. Later I'll add there more prototypes into libmariadb/mariadb_priv.h, e.g. functions that will be added under terms of MDEV-17832. Note, I also removed the "my_bool long_flag_protocol" parameter from unpack_fields(), it's not used anyway. Under terms of MDEV-17832 I'll have to add a "const MYSQL *mysql" parameter. So if we ever need to check CLIENT_LONG_FLAG inside unpack_fields() again, it can be done through the "mysql" parameter: (mysql->server_capabilities & CLIENT_LONG_FLAG) Thanks!

1 0

Re: [Maria-developers] 9d2c63d45e4: MDEV-20076: SHOW GRANTS does not quote role names properly
by Sergei Golubchik 19 Jan '20

19 Jan '20

Hi, Oleksandr! On Jan 19, Oleksandr Byelkin wrote: > revision-id: 9d2c63d45e4 (mariadb-10.3.20-6-g9d2c63d45e4) > parent(s): d4edb0510ec > author: Oleksandr Byelkin <sanja(a)mariadb.com> > committer: Oleksandr Byelkin <sanja(a)mariadb.com> > timestamp: 2019-11-14 09:32:54 +0100 > message: > > MDEV-20076: SHOW GRANTS does not quote role names properly > > Quotes added to output. > > diff --git a/mysql-test/main/grant5.test b/mysql-test/main/grant5.test > index 649bba7d1ca..045cbf8fc86 100644 > --- a/mysql-test/main/grant5.test > +++ b/mysql-test/main/grant5.test > @@ -33,3 +33,26 @@ REVOKE EXECUTE ON PROCEDURE sp FROM u; > --error ER_TABLE_NOT_LOCKED > REVOKE PROCESS ON *.* FROM u; > DROP TABLE t1; > + > +--echo # > +--echo # MDEV-20076: SHOW GRANTS does not quote role names properly > +--echo # > + > +create role 'role-1'; > +create role 'rock\'n\'roll'; > +create user 'user-1'@'localhost'; > +create user 'O\'Brien'@'localhost'; > +grant select on mysql.user to 'role-1'; > +grant select on mysql.user to 'rock\'n\'roll'; > +GRANT 'role-1' TO 'user-1'@'localhost'; > +GRANT 'rock\'n\'roll' TO 'O\'Brien'@'localhost'; > +show grants for 'role-1'; > +show grants for 'rock\'n\'roll'; > +show grants for 'user-1'@'localhost'; > +show grants for 'O\'Brien'@'localhost'; > +drop role 'role-1'; > +drop role 'rock\'n\'roll'; > +drop user 'user-1'@'localhost'; > +drop user 'O\'Brien'@'localhost'; > + > +--echo # End of 10.3 tests please add few tests when sql_quote_show_create=0. Like 1. sql_quote_show_create=0, role name is not reserved (e.g. foo) 2. sql_quote_show_create=0, role name is reserved (e.g. fetch) 3. sql_quote_show_create=0, role name is not a valid identifier (e.g. role-1) Then ok to push. Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

Re: [Maria-developers] edc9059c31b: MDEV-18027: Running out of file descriptors and eventual crash
by Sergei Golubchik 19 Jan '20

19 Jan '20

Hi, Oleksandr! Ok to push. Just one comment, see below: On Jan 19, Oleksandr Byelkin wrote: > revision-id: edc9059c31b (mariadb-10.2.28-3-gedc9059c31b) > parent(s): 0b1bc4bf76f > author: Oleksandr Byelkin <sanja(a)mariadb.com> > committer: Oleksandr Byelkin <sanja(a)mariadb.com> > timestamp: 2019-11-05 16:24:26 +0100 > message: > > MDEV-18027: Running out of file descriptors and eventual crash > > For automatic number of opened files limit take into account number of > table instances for table cache > > diff --git a/sql/mysqld.cc b/sql/mysqld.cc > index 34e5704bcfe..9907c17ee07 100644 > --- a/sql/mysqld.cc > +++ b/sql/mysqld.cc > @@ -4461,6 +4461,17 @@ static int init_common_variables() > if (files < wanted_files && global_system_variables.log_warnings) > sql_print_warning("Could not increase number of max_open_files to more than %u (request: %u)", files, wanted_files); > > + /* > + If we required too much tc_instances than we reduce it but not more > + then in half The comment is wrong, you don't do "not more than in half", you can reduce it all the way to 1. Which, I think, it correct. So the code is ok, but the comment is misleading, please remove the "but not more than in half" part. > + */ > + SYSVAR_AUTOSIZE_IF_CHANGED(tc_instances, > + (uint32) MY_MIN(MY_MAX((files - extra_files - > + max_connections)/ > + 2/tc_size, > + 1), > + tc_instances), > + uint32); > /* > If we have requested too much file handles than we bring > max_connections in supported bounds. Still leave at least Regards, Sergei VP of MariaDB Server Engineering and security(a)mariadb.org

1 0

[Maria-developers] 回复: [Maria-discuss] [MariaDB 10.3.7] Hello, everyone. My process core dumped while calling mysql_ping().
by Ma Allen 15 Jan '20

15 Jan '20

Hi Markus, Only MySqlWaitConnected() in the two functions doesn't use lock, reading database uses mutex. What's more, the two functions in initialization relates to ONLY the main thread. No other threads call them. Previously, the program is written with mysqllib, which is not MariaDB lib currently used. Mysqllib supports multiple threads to concurrently use one connection. Its tutorials were also referred to: https://dev.mysql.com/doc/refman/5.7/en/c-api-threaded-clients.html MySQL :: MySQL 5.7 Reference Manual :: 27.7.3.3 Writing C API Threaded Client Programs<https://dev.mysql.com/doc/refman/5.7/en/c-api-threaded-clients.html> This section provides guidance for writing client programs that use the thread-related functions in the MySQL C API. For further information about these functions, see Section 27.7.11, “C API Threaded Function Descriptions”.For examples of source code that uses them, look in the client directory of a MySQL source distribution: dev.mysql.com After replacing mysqllib with MariaDB lib, everything works well on the test environment except one of the clients'. Allen ________________________________ 发件人: Maria-discuss <maria-discuss-bounces+mazhh=outlook.com(a)lists.launchpad.net> 代表 Markus Mäkelä <markus.makela(a)mariadb.com> 发送时间: 2020年1月14日 15:40 收件人: maria-discuss(a)lists.launchpad.net <maria-discuss(a)lists.launchpad.net> 主题: Re: [Maria-discuss] [Maria-developers] [MariaDB 10.3.7] Hello, everyone. My process core dumped while calling mysql_ping(). Hi, Is the connection protected by a lock of some sorts? To my knowledge the library does not support concurrent use of a single connection. I'd recommend trying to first wrap all access to the connection with a mutex and see if that solves the problem. Markus On 1/14/20 08:33, Ma Allen wrote: Environment: MariaDB 10.3.7 database server & related headers and libraries CentOS 7.3 or 7.4 ./configure --prefix=/home/spiderflow --localstatedir=/home/spiderflow --enable-unix-socket --with-libnss-libraries=/usr/lib64 --with-libnss-includes=/usr/include/nss3 --with-libnspr-libraries=/usr/lib64 --with-libnspr-includes=/usr/include/nspr4 --enable-netmap --enable-rust --disable-gccmarch-native --enable-mysql --with-libmysql-includes=/usr/local/mariadb-10.3.7-linux-systemd-x86_64/include/mysql --with-libmysql-libraries=/usr/local/mariadb-10.3.7-linux-systemd-x86_64/lib Phenomenon: (gdb) #0 ma_pvio_write (pvio=0x10101010101b2e0, buffer=buffer@entry=0x2510101 "\001", length=length@entry=5) at /home/buildbot/buildbot/build/libmariadb/libmariadb/ma_pvio.c:350 #1 0x00007f7100249dea in ma_net_real_write (net=net@entry=0xee9de0 <g_mysql_info>, packet=0x2510101 "\001", len=5) at /home/buildbot/buildbot/build/libmariadb/libmariadb/ma_net.c:335 #2 0x00007f7100249f60 in ma_net_flush (net=net@entry=0xee9de0 <g_mysql_info>) at /home/buildbot/buildbot/build/libmariadb/libmariadb/ma_net.c:166 #3 0x00007f710024a71a in ma_net_write_command (net=net@entry=0xee9de0 <g_mysql_info>, command=command@entry=14 '\016', packet=packet@entry=0x7f71003d6c67 "", len=0, disable_flush=disable_flush@entry=0 '\000') at /home/buildbot/buildbot/build/libmariadb/libmariadb/ma_net.c:244 #4 0x00007f7100252ca4 in mthd_my_send_cmd (mysql=0xee9de0 <g_mysql_info>, command=<optimized out>, arg=0x7f71003d6c67 "", length=0, skipp_check=<optimized out>, opt_arg=<optimized out>) at /home/buildbot/buildbot/build/libmariadb/libmariadb/mariadb_lib.c:394 #5 0x00007f71002518d0 in mysql_ping (mysql=0xee9de0 <g_mysql_info>) at /home/buildbot/buildbot/build/libmariadb/libmariadb/mariadb_lib.c:2552 #6 0x00000000006651b2 in MySqlWaitConnected (p_mysql_data=p_mysql_data@entry=0xee9de0 <g_mysql_info>) at ../srcSF/spiderFlow-proto-baseline.c:676 #7 0x0000000000668819 in LoadProtoConfInfos (pInfo=pInfo@entry=0xee9de0 <g_mysql_info>) at ../srcSF/spiderFlow-proto-baseline.c:835 #8 0x000000000066c53a in InitBaseLine () at ../srcSF/spiderFlow-proto-baseline.c:1069 #9 0x00000000005f308b in PostConfLoadedSetup (suri=0x10dbb80 <suricata>) at suricata.c:2900 #10 0x0000000000425ce8 in main (argc=5, argv=0x7fff1cdca258) at suricata.c:3072 (gdb) Analyzation: The core dump always exists after running for a while or maybe after the sever restarts. It occurs at the initialization and involves with ONLY ONE thread, which is the main thread of the whole process. The related code is two functions, which both checks the connection and read data from MariaDB. The first function works fine, however, the second one's checking connection leads to core dump. The code to check connection is as follows: void MySqlWaitConnected(MYSQL * p_mysql_data) { while (mysql_ping(p_mysql_data) != 0) { SCLogInfo("Mysql ping error:%s", mysql_error(p_mysql_data)); #ifdef WIN32 Sleep(3000); #else sleep(3); #endif } } Previously, I solved a likely similar problem as follows: Multiple threads shared ONE MySQL handler, which is the MYSQL data structure. Before inserting data into MariaDB without mutex, MySqlWaitConnected() is called. So while multiple threads do the same operation, inserting data is interrupted by mysql_ping() that leads to core dump. Afterwards, I comment MySqlWaitConnected() and add mutex for mysql_real_query() and mysql_commit(). Does anyone meet the same problem before? Any suggestion is appreciated. Thanks in advance. Best Regards, Allen Ma _______________________________________________ Mailing list: https://launchpad.net/~maria-developers Post to : maria-developers(a)lists.launchpad.net<mailto:maria-developers@lists.launchpad.net> Unsubscribe : https://launchpad.net/~maria-developers More help : https://help.launchpad.net/ListHelp -- Markus Mäkelä, Senior Software Engineer MariaDB Corporation t: +358 40 7740484 | IRC: markusjm@freenode

1 2