- developers - lists.mariadb.org

Re: [Maria-developers] [Commits] 4861977ce76: MDEV-14695: Assertion `n < m_size' failed in Bounds_checked_array<Element_type>::operator
by Vicențiu Ciorbaru 14 May '18

14 May '18

Hi Varun! I will take a second look at this but a first minor comment is to not save two characters by shortening fields to flds, it makes reading unnatural. Vicentiu On Sat, 12 May 2018 at 21:45 Varun <varunraiko1803(a)gmail.com> wrote: > revision-id: 4861977ce76761fefb9779a774984b6afa818f2e > (mariadb-10.2.5-636-g4861977ce76) > parent(s): 8b26fea83572cf5c043721b7835c3828937f9c27 > author: Varun Gupta > committer: Varun Gupta > timestamp: 2018-05-13 00:09:15 +0530 > message: > > MDEV-14695: Assertion `n < m_size' failed in > Bounds_checked_array<Element_type>::operator > > In this issue we hit the assert because we are adding addition fields to > the field JOIN::all_fields list. This > is done because HEAP tables can't index BIT fields so we need to use an > additional hidden field for grouping because later it will be > converted to a LONG field. Original field will remain of the BIT type and > will be returned. This happens when we convert DISTINCT to > GROUP BY. > > The solution is to take into account the number of such hidden fields that > would be added to the field JOIN::all_fields list while calculating > the size of the ref_pointer_array. > > --- > sql/sql_base.cc | 9 ++++++--- > sql/sql_base.h | 4 ++-- > sql/sql_delete.cc | 3 ++- > sql/sql_lex.cc | 8 +++++++- > sql/sql_lex.h | 5 +++++ > sql/sql_select.cc | 4 +++- > sql/sql_union.cc | 1 + > 7 files changed, 26 insertions(+), 8 deletions(-) > > diff --git a/sql/sql_base.cc b/sql/sql_base.cc > index 95ff41cb6f4..81ccfe5e514 100644 > --- a/sql/sql_base.cc > +++ b/sql/sql_base.cc > @@ -6919,7 +6919,7 @@ static bool setup_natural_join_row_types(THD *thd, > > int setup_wild(THD *thd, TABLE_LIST *tables, List<Item> &fields, > List<Item> *sum_func_list, > - uint wild_num) > + uint wild_num, uint *hidden_bit_flds) > { > if (!wild_num) > return(0); > @@ -6960,7 +6960,7 @@ int setup_wild(THD *thd, TABLE_LIST *tables, > List<Item> &fields, > else if (insert_fields(thd, ((Item_field*) item)->context, > ((Item_field*) item)->db_name, > ((Item_field*) item)->table_name, &it, > - any_privileges)) > + any_privileges, hidden_bit_flds)) > { > if (arena) > thd->restore_active_arena(arena, &backup); > @@ -7425,7 +7425,7 @@ bool get_key_map_from_key_list(key_map *map, TABLE > *table, > bool > insert_fields(THD *thd, Name_resolution_context *context, const char > *db_name, > const char *table_name, List_iterator<Item> *it, > - bool any_privileges) > + bool any_privileges, uint *hidden_bit_flds) > { > Field_iterator_table_ref field_iterator; > bool found; > @@ -7545,6 +7545,9 @@ insert_fields(THD *thd, Name_resolution_context > *context, const char *db_name, > else > it->after(item); /* Add 'item' to the SELECT list. */ > > + if (item->type() == Item::FIELD_ITEM && item->field_type() == > MYSQL_TYPE_BIT) > + (*hidden_bit_flds)++; > + > #ifndef NO_EMBEDDED_ACCESS_CHECKS > /* > Set privilege information for the fields of newly created views. > diff --git a/sql/sql_base.h b/sql/sql_base.h > index d6063f1b771..8d0dcb2f197 100644 > --- a/sql/sql_base.h > +++ b/sql/sql_base.h > @@ -154,11 +154,11 @@ bool fill_record_n_invoke_before_triggers(THD *thd, > TABLE *table, > enum trg_event_type event); > bool insert_fields(THD *thd, Name_resolution_context *context, > const char *db_name, const char *table_name, > - List_iterator<Item> *it, bool any_privileges); > + List_iterator<Item> *it, bool any_privileges, uint > *hidden_bit_flds); > void make_leaves_list(THD *thd, List<TABLE_LIST> &list, TABLE_LIST > *tables, > bool full_table_list, TABLE_LIST *boundary); > int setup_wild(THD *thd, TABLE_LIST *tables, List<Item> &fields, > - List<Item> *sum_func_list, uint wild_num); > + List<Item> *sum_func_list, uint wild_num, uint * > hidden_bit_flds); > bool setup_fields(THD *thd, Ref_ptr_array ref_pointer_array, > List<Item> &item, enum_mark_columns mark_used_columns, > List<Item> *sum_func_list, List<Item> *pre_fix, > diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc > index 7b7a7e3f804..7bae7736cf3 100644 > --- a/sql/sql_delete.cc > +++ b/sql/sql_delete.cc > @@ -760,7 +760,8 @@ l > select_lex->leaf_tables, FALSE, > DELETE_ACL, SELECT_ACL, TRUE)) > DBUG_RETURN(TRUE); > - if ((wild_num && setup_wild(thd, table_list, field_list, NULL, > wild_num)) || > + if ((wild_num && setup_wild(thd, table_list, field_list, NULL, wild_num, > + &select_lex->hidden_bit_flds)) || > setup_fields(thd, Ref_ptr_array(), > field_list, MARK_COLUMNS_READ, NULL, NULL, 0) || > setup_conds(thd, table_list, select_lex->leaf_tables, conds) || > diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc > index 3e36cac96b9..49150c863fe 100644 > --- a/sql/sql_lex.cc > +++ b/sql/sql_lex.cc > @@ -2143,6 +2143,7 @@ void st_select_lex::init_query() > m_agg_func_used= false; > window_specs.empty(); > window_funcs.empty(); > + hidden_bit_flds= 0; > } > > void st_select_lex::init_select() > @@ -2673,6 +2674,10 @@ ulong st_select_lex::get_table_join_options() > > bool st_select_lex::setup_ref_array(THD *thd, uint order_group_num) > { > + > + if (!((options & SELECT_DISTINCT) && !group_list)) > + hidden_bit_flds= 0; > + > // find_order_in_list() may need some extra space, so multiply by two. > order_group_num*= 2; > > @@ -2687,7 +2692,8 @@ bool st_select_lex::setup_ref_array(THD *thd, uint > order_group_num) > select_n_reserved + > select_n_having_items + > select_n_where_fields + > - order_group_num) * 5; > + order_group_num + > + hidden_bit_flds) * 5; > if (!ref_pointer_array.is_null()) > { > /* > diff --git a/sql/sql_lex.h b/sql/sql_lex.h > index 6d281930d1f..8283496604e 100644 > --- a/sql/sql_lex.h > +++ b/sql/sql_lex.h > @@ -956,6 +956,11 @@ class st_select_lex: public st_select_lex_node > > /* it is for correct printing SELECT options */ > thr_lock_type lock_type; > + /* > + it counts the number of bit fields in the SELECT list. These are used > when DISTINCT is > + converted to a GROUP BY involving BIT fields. > + */ > + uint hidden_bit_flds; > > void init_query(); > void init_select(); > diff --git a/sql/sql_select.cc b/sql/sql_select.cc > index 4cd4754596b..83953661ce3 100644 > --- a/sql/sql_select.cc > +++ b/sql/sql_select.cc > @@ -799,7 +799,9 @@ JOIN::prepare(TABLE_LIST *tables_init, > select_lex != select_lex->master_unit()->global_parameters()) > real_og_num+= select_lex->order_list.elements; > > - if (setup_wild(thd, tables_list, fields_list, &all_fields, wild_num)) > + DBUG_ASSERT(select_lex->hidden_bit_flds == 0); > + if (setup_wild(thd, tables_list, fields_list, &all_fields, wild_num, > + &select_lex->hidden_bit_flds)) > DBUG_RETURN(-1); > if (select_lex->setup_ref_array(thd, real_og_num)) > DBUG_RETURN(-1); > diff --git a/sql/sql_union.cc b/sql/sql_union.cc > index 3fe8a24f8bd..42904e341b6 100644 > --- a/sql/sql_union.cc > +++ b/sql/sql_union.cc > @@ -1474,6 +1474,7 @@ bool st_select_lex::cleanup() > } > inner_refs_list.empty(); > exclude_from_table_unique_test= FALSE; > + hidden_bit_flds= 0; > DBUG_RETURN(error); > } > > _______________________________________________ > commits mailing list > commits(a)mariadb.org > https://lists.askmonty.org/cgi-bin/mailman/listinfo/commits

1 0

[Maria-developers] MDEV-6111: Optimizer trace: features and milestones
by Sergey Petrunia 11 May '18

11 May '18

Hi Zhzhzoo, Please find below some thoughts about features/milestones in the project. Items at the top of the list should be done first. The list is not necesarrily complete, any comments/additions are welcome. - Optimizer trace collection API (is it different from Json_writer that we have? Does MySQL use the same API to produce EXPLAIN FORMAT=JSON and produce optimizer_trace? Should we follow them?) - Basic optimizer trace collection (just print the query and "hello":"world" instead of trace). Clear the trace data at query start, collect the trace if tracing is switched on (and do not collect if it is off) - INFORMATION_SCHEMA.OPTIMIZER_TRACE table in I_S database. Produce optimizer_trace content for features common with MySQL: = condition rewrite = partition pruning = range analysis = ref/eq_ref analysis (update_ref_and_keys) = (TODO what else?) Produce optimizer_trace content for MariaDB-specific features: = merged semi-join conversion = non-merged semi-join conversion = join condition pushdown = (TODO what else?) - optimizer_trace_limit/optimizer_trace_offset (without this feature, just save one last trace) - Ability to produce either a pretty-printed or one-line optimizer trace - Security = INFORMATION_SCHEMA.OPTIMIZER_TRACE.INSUFFICIENT_PRIVILEGES = Who can view optimizer_trace? = Figure out and implement the necessary security restrictions - Memory limits = Honor the @@optimizer_trace_max_mem_size setting. BR Sergei -- Sergei Petrunia, Software Developer MariaDB Corporation | Skype: sergefp | Blog: http://s.petrunia.net/blog

2 1

[Maria-developers] Please review MDEV-16134 Wrong I_S.COLUMNS.CHARACTER_XXX_LENGTH value for compressed columns
by Alexander Barkov 10 May '18

10 May '18

Hi Sergey, Please review a patch for MDEV-16134. Thanks!

1 0

[Maria-developers] MDEV-15935: Connection Redirection Mechanism in MariaDB Client/Server Protocol
by xiangyu hu 09 May '18

09 May '18

Hi developers, There is an ongoing discussion regarding to the proposal of connection redirection in JIRA. Please kindly take a look if you aren't aware of it. Link is: https://jira.mariadb.org/browse/MDEV-15935. Thank you, Xiangyu

1 0

[Maria-developers] MDEV-15450: Add Host Name Field to MariaDB Client Handshake Protocol
by xiangyu hu 09 May '18

09 May '18

Hello devs, We have initiated a proposal in JIRA where we described the necessity of adding host_name into login packet attributes. I would like to broaden the audience here to bring up more discussions. Please take a look at MDEV-15450 (https://jira.mariadb.org/browse/MDEV-15450) [MDEV-15450] Add Host Name Field to MariaDB Client ...<https://jira.mariadb.org/browse/MDEV-15450> jira.mariadb.org Why. In original, one MariaDB server instance is identified by one address (IP + port). That is not enough for a shared proxy for multi servers. Thanks, Xiangyu

1 0

Re: [Maria-developers] eb3e9cd: MDEV-15813 ASAN use-after-poison in hp_hashnr upon HANDLER READ on a versioned HEAP table.
by Sergei Golubchik 09 May '18

09 May '18

Hi, Alexey! On May 08, Alexey Botchkov wrote: > revision-id: eb3e9cdbe86b5a68c3ec8a9f26165259e4ca7eec (mariadb-10.3.6-125-geb3e9cd) > parent(s): bd09c5ca866e273b6cebbd9a15c51c82bfa3ac9b > committer: Alexey Botchkov > timestamp: 2018-05-08 14:19:55 +0400 > message: > > MDEV-15813 ASAN use-after-poison in hp_hashnr upon HANDLER READ on a versioned HEAP table. > > Returns an error if we try to use a key in a query it declared unable to > handle. > > diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt > index c55ac4f..7446a60 100644 > --- a/sql/share/errmsg-utf8.txt > +++ b/sql/share/errmsg-utf8.txt > @@ -7915,3 +7915,5 @@ ER_UPDATED_COLUMN_ONLY_ONCE > eng "The column %`s.%`s cannot be changed more than once in a single UPDATE statement" > ER_EMPTY_ROW_IN_TVC > eng "Row with no elements is not allowed in table value constructor in this context" > +ER_KEY_DOESNT_SUPPORT > + eng "%s index %`s does not support this operation" I think this can be safely pushed into 10.2. But not earlier, we can only add new error messages in the latest GA version, not in previos GA's. > diff --git a/sql/sql_handler.cc b/sql/sql_handler.cc > index 09883d8..12e4b5a 100644 > --- a/sql/sql_handler.cc > +++ b/sql/sql_handler.cc > @@ -661,12 +661,30 @@ mysql_ha_fix_cond_and_key(SQL_HANDLER *handler, > key_part_map keypart_map; > uint key_len; > > + if (ha_rkey_mode != HA_READ_KEY_EXACT && > + (table->file->index_flags(handler->keyno, 0, TRUE) & > + (HA_READ_NEXT | HA_READ_PREV | HA_READ_RANGE)) == 0) I suspect there can be hypothetical cases where an index supports some specific ha_rkey_mode values and has some specific flags, but not all of them, like you test here. But as the only real use case is HASH vs BTREE now, your check is fine. No need overengineer a complex test that we cannot possibly test now. > + { > + my_error(ER_KEY_DOESNT_SUPPORT, MYF(0), > + table->file->index_type(handler->keyno), keyinfo->name); > + return 1; > + } > + > if (key_expr->elements > keyinfo->user_defined_key_parts) > { > my_error(ER_TOO_MANY_KEY_PARTS, MYF(0), > keyinfo->user_defined_key_parts); > return 1; > } > + else if (key_expr->elements < keyinfo->user_defined_key_parts && > + (table->file->index_flags(handler->keyno, 0, TRUE) & > + HA_ONLY_WHOLE_INDEX)) > + { > + my_error(ER_KEY_DOESNT_SUPPORT, MYF(0), > + table->file->index_type(handler->keyno), keyinfo->name); > + return 1; Good point! > + } This is ok to push. Just two questions: 1. Why did you add it in sql_handler.cc and not in the class handler method, like ha_index_read_map or ha_index_next ? 2. Could you add tests for fulltext/rtree indexes too, please? Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

1 0

[Maria-developers] GSoC 2018 - MDEV-12313 - 30th of April -> 6th of May Update
by Teodor-Vicentiu Niculescu 08 May '18

08 May '18

Greetings, First of all, I want to thank you for accepting me for this years GSoC. I plan to do my best during the following months in order to implement the SQL Statement specified in the proposal. https://docs.google.com/document/d/1dA7YlDPlRG43LhfGlHXnHJiERfjExLG9qS7BqXh… In the last week (30th of April to 6th of May) I have started the first task of the proposal, specifically extending the grammar. After receiving some initial hints from Mr. Ciorbaru, concerning where to start looking in the files ( ANALYZE_SYM in sql/sql_yacc.yy ), I quickly discovered I have no knowledge of GNU Bison. I took my time to look through the GNU Bison online documentation, I have compiled tests and written code in order to better understand parsing. Now, sql_yacc.yy is starting to make sense, which is why I plan to finish the parsing part of the project by Sunday (13th of May). Until 15th of June my schedule will be quite busy because finals are approaching. You can still contact me by e-mail. Advice and constructive criticism is always much appreciated. Thank you for this amazing opportunity. Sincerely, Teodor

1 0

Re: [Maria-developers] fa0b71b0e19: MDEV-10259 mysqld crash with certain statement length and...
by Sergei Golubchik 08 May '18

08 May '18

Hi, Sachin! On May 08, Sachin Setiya wrote: > diff --git a/unittest/sql/mf_iocache-t.cc b/unittest/sql/mf_iocache-t.cc > index 31f98562521..d5e2ffb095e 100644 > --- a/unittest/sql/mf_iocache-t.cc > +++ b/unittest/sql/mf_iocache-t.cc > @@ -187,10 +187,71 @@ void mdev9044() > close_cached_file(&info); > } > > +/* 2 Reads in cache makes second read to fail (only if first read uses temp file) */ > +void mdev10259() > +{ > + int res; > + uchar buf[CACHE_SIZE + 200]; > + memset(buf, FILL, sizeof(buf)); > + > + diag("MDEV-10259- mysqld crash with certain statement length and order with" > + " Galera and encrypt-tmp-files=1"); > + > + init_io_cache_encryption(); > + > + res= open_cached_file(&info, 0, 0, CACHE_SIZE, 0); > + ok(res == 0, "open_cached_file" INFO_TAIL); > + > + res= my_b_write(&info, buf, sizeof(buf)); > + ok(res == 0 && info.pos_in_file == CACHE_SIZE, "large write" INFO_TAIL); > + > + res= my_b_flush_io_cache(&info, 1); > + ok(res == 0, "flush" INFO_TAIL); I think this misses the point. The bug happened when the data is in the cache, and there's no file to read from, so when the data is lost - it's forever. But my_b_flush_io_cache() writes the buffer to a file, so my_b_fill() can read it even if the buffer is emptied. Meaning, I suspect that if you remove your fix from mf_iocache_encr.cc, this unit test will still pass. Please double-check that your test fails without a fix. Ok to push after that. > + res= reinit_io_cache(&info, READ_CACHE, 0, 0, 0); > + ok(res == 0, "reinit READ_CACHE" INFO_TAIL); Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

1 0

Re: [Maria-developers] 7107565: MDEV-10259 mysqld crash with certain statement length and...
by Sergei Golubchik 08 May '18

08 May '18

Hi, Sachin! Great work! On Mar 27, Sachin Setiya wrote: > revision-id: 710756527ec8589bce33bc834a1d8e6604266b6d (mariadb-10.1.31-47-g7107565) > parent(s): 8152c52e1af608da5d288a4e6fc82e2d804e64cd > author: Sachin Setiya > committer: Sachin Setiya > timestamp: 2018-03-27 04:24:57 +0530 > message: > > MDEV-10259 mysqld crash with certain statement length and... > order with Galera and encrypt-tmp-files=1 > > Problem:- If trans_cache (IO_CACHE) uses encrypted tmp file > then on next DML server will crash. > > Case:- > Lets take a case , we have a table t1 , We try to do 2 inserts in t1 > 1. A really long insert so that trans_cache has to use temp_file > 2. Just a small insert > > Analysis:- Actually server crashes from inside of galera > library. > /lib64/libc.so.6(abort+0x175)[0x7fb5ba779dc5] > /usr/lib64/galera/libgalera_smm.so(_ZN6galera3FSMINS_9TrxHandle5State... > mysys/stacktrace.c:247(my_print_stacktrace)[0x7fb5a714940e] > sql/signal_handler.cc:160(handle_fatal_signal)[0x7fb5a715c1bd] > sql/wsrep_hton.cc:257(wsrep_rollback)[0x7fb5bcce923a] > sql/wsrep_hton.cc:268(wsrep_rollback)[0x7fb5bcce9368] > sql/handler.cc:1658(ha_rollback_trans(THD*, bool))[0x7fb5bcd4f41a] > sql/handler.cc:1483(ha_commit_trans(THD*, bool))[0x7fb5bcd4f804] > > but actual issue is not in galera but in mariadb, because for 2nd > insert we should never call rollback. We are calling rollback because > log_and_order fails it fails because write_cache fails , It fails > because after reinit_io_cache(trans_cache) , my_b_tell says 0 so we > look into tmp_file for data , which is obviously wrong since temp > was used for previous insert and it no longer exist. Add here something like: wsrep_write_cache_inc() reads the IO_CACHE in a loop, filling it with my_b_fill() until it returns "0 bytes read". Later MYSQL_BIN_LOG::write_cache() does the same. wsrep_write_cache_inc() assumes that reading a zero bytes past EOF leaves the old data in the cache. Let's do that. > diff --git a/sql/mf_iocache_encr.cc b/sql/mf_iocache_encr.cc > index 149e6fe..41e1333 100644 > --- a/sql/mf_iocache_encr.cc > +++ b/sql/mf_iocache_encr.cc > @@ -49,8 +49,7 @@ static int my_b_encr_read(IO_CACHE *info, uchar *Buffer, size_t Count) > > if (pos_in_file == info->end_of_file) > { here, a comment: /* reading past EOF should not empty the cache */ > - info->read_pos= info->read_end= info->buffer; > - info->pos_in_file= pos_in_file; > + info->read_pos= info->read_end; > info->error= 0; > DBUG_RETURN(MY_TEST(Count)); > } and add a test for it into unittest/sql/mf_iocache-t.cc. Something simple, like, write into a cache, reinit to READ, fill, fill, reinit to READ, verify the data is still there. Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

2 2

Re: [Maria-developers] bea915f: MDEV-15965 Invisible columns and LOAD DATA don't work well together:...
by Sergei Golubchik 07 May '18

07 May '18

Hi, sachin! On Apr 26, sachin wrote: > revision-id: bea915f21a2a2771ce7f9e19dcb9d6893f33f7bd (mariadb-10.3.6-47-gbea915f) > parent(s): 326db1a2aaa9b275a1a21a863e8cd2d9fa1b1d5f > author: Sachin Setiya > committer: Sachin Setiya > timestamp: 2018-04-26 16:49:27 +0530 > message: > > MDEV-15965 Invisible columns and LOAD DATA don't work well together:... > ER_WARN_TOO_FEW_RECORDS > > Fix mysql_load iterator to skip invisible fields. > > diff --git a/sql/sql_load.cc b/sql/sql_load.cc > index cfa92f1..fbb3926 100644 > --- a/sql/sql_load.cc > +++ b/sql/sql_load.cc > @@ -444,6 +444,8 @@ int mysql_load(THD *thd, const sql_exchange *ex, TABLE_LIST *table_list, > field_iterator.set(table_list); > for (; !field_iterator.end_of_fields(); field_iterator.next()) > { > + if (field_iterator.field()->invisible > VISIBLE) > + continue; Oh, that's all? And the syntax with SET just worked automatically? Cool. ok to push! Regards, Sergei Chief Architect MariaDB and security(a)mariadb.org

1 0