- commits - lists.mariadb.org

[Commits] 5ce5b99837b: MDEV-17255: New optimizer defaults and ANALYZE TABLE
by Varun 05 Dec '18

05 Dec '18

revision-id: 5ce5b99837b70fa72698695f15d62d732a112644 (mariadb-10.3.6-266-g5ce5b99837b) parent(s): e0739064450f2c2be6d5de1d799582121747dd39 author: Varun Gupta committer: Varun Gupta timestamp: 2018-12-05 18:49:49 +0530 message: MDEV-17255: New optimizer defaults and ANALYZE TABLE Added to new values to the server variable use_stat_tables. The values are COMPLEMENTARY_FOR_QUERIES and PREFERABLY_FOR_QUERIES. Both these values don't allow to collect EITS for queries like analyze table t1; To collect EITS we would need to use the syntax with persistent like analyze table t1 persistent for columns (col1,col2...) index (idx1, idx2...) / ALL Changing the default value from NEVER to PREFERABLY_FOR_QUERIES. --- mysql-test/include/default_mysqld.cnf | 1 + mysql-test/main/stat_tables.result | 59 +++++++++++++++++++++++++++++++ mysql-test/main/stat_tables.test | 43 ++++++++++++++++++++++ mysql-test/main/stat_tables_innodb.result | 59 +++++++++++++++++++++++++++++++ sql/sql_admin.cc | 2 +- sql/sql_statistics.cc | 5 ++- sql/sql_statistics.h | 15 ++++++++ sql/sys_vars.cc | 5 +-- 8 files changed, 183 insertions(+), 6 deletions(-) diff --git a/mysql-test/include/default_mysqld.cnf b/mysql-test/include/default_mysqld.cnf index 69a2b58288b..5ba3bdeb92c 100644 --- a/mysql-test/include/default_mysqld.cnf +++ b/mysql-test/include/default_mysqld.cnf @@ -107,6 +107,7 @@ loose-performance-schema-consumer-thread-instrumentation=ON binlog-direct-non-transactional-updates default-storage-engine=myisam +use-stat-tables=preferably_for_queries loose-ssl-ca=(a)ENV.MYSQL_TEST_DIR/std_data/cacert.pem loose-ssl-cert=(a)ENV.MYSQL_TEST_DIR/std_data/server-cert.pem diff --git a/mysql-test/main/stat_tables.result b/mysql-test/main/stat_tables.result index 308529ece47..67bf3f9237d 100644 --- a/mysql-test/main/stat_tables.result +++ b/mysql-test/main/stat_tables.result @@ -605,4 +605,63 @@ SELECT MAX(pk) FROM t1; MAX(pk) NULL DROP TABLE t1; +# +# MDEV-17255: New optimizer defaults and ANALYZE TABLE +# +set @save_use_stat_tables= @@use_stat_tables; +create table t1 (a int, b int); +insert into t1(a,b) values (1,2),(1,3),(1,4),(1,5),(2,6),(2,7),(3,8),(3,9),(3,9),(4,10); +# +# with use_stat_tables= PREFERABLY_FOR_QUERIES +# analyze table t1 will not collect statistics +# +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +select * from mysql.column_stats; +db_name table_name column_name min_value max_value nulls_ratio avg_length avg_frequency hist_size hist_type histogram +test t1 a 1 4 0.0000 4.0000 2.5000 0 NULL NULL +test t1 b 2 10 0.0000 4.0000 1.1111 0 NULL NULL +analyze +select * from t1 where a = 1 and b=3; +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 10 10.00 2.78 10.00 Using where +# +# with use_stat_tables= PREFERABLY_FOR_QUERIES +# analyze table t1 will collect statistics if we use PERSISTENT +# for columns, indexes or everything +# +analyze table t1 persistent for columns (a) indexes (); +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status Table is already up to date +select * from mysql.column_stats; +db_name table_name column_name min_value max_value nulls_ratio avg_length avg_frequency hist_size hist_type histogram +test t1 a 1 4 0.0000 4.0000 2.5000 0 NULL NULL +test t1 b 2 10 0.0000 4.0000 1.1111 0 NULL NULL +# filtered shows that we used the data from stat tables +analyze +select * from t1 where a = 1 and b=3; +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 10 10.00 2.78 10.00 Using where +# +# with use_stat_tables= PREFERABLY +# analyze table t1 will collect statistics +# +set @@use_stat_tables=PREFERABLY; +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status Table is already up to date +select * from mysql.column_stats; +db_name table_name column_name min_value max_value nulls_ratio avg_length avg_frequency hist_size hist_type histogram +test t1 a 1 4 0.0000 4.0000 2.5000 0 NULL NULL +test t1 b 2 10 0.0000 4.0000 1.1111 0 NULL NULL +# filtered shows that we used the data from stat tables +analyze +select * from t1 where a=1 and b=3; +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 10 10.00 2.78 10.00 Using where +drop table t1; set use_stat_tables=@save_use_stat_tables; diff --git a/mysql-test/main/stat_tables.test b/mysql-test/main/stat_tables.test index 19bc0fa2f46..8616f6386e1 100644 --- a/mysql-test/main/stat_tables.test +++ b/mysql-test/main/stat_tables.test @@ -384,4 +384,47 @@ SELECT MAX(pk) FROM t1; DROP TABLE t1; +--echo # +--echo # MDEV-17255: New optimizer defaults and ANALYZE TABLE +--echo # + +set @save_use_stat_tables= @@use_stat_tables; +create table t1 (a int, b int); +insert into t1(a,b) values (1,2),(1,3),(1,4),(1,5),(2,6),(2,7),(3,8),(3,9),(3,9),(4,10); + +--echo # +--echo # with use_stat_tables= PREFERABLY_FOR_QUERIES +--echo # analyze table t1 will not collect statistics +--echo # + +analyze table t1; +select * from mysql.column_stats; +analyze +select * from t1 where a = 1 and b=3; + +--echo # +--echo # with use_stat_tables= PREFERABLY_FOR_QUERIES +--echo # analyze table t1 will collect statistics if we use PERSISTENT +--echo # for columns, indexes or everything +--echo # + +analyze table t1 persistent for columns (a) indexes (); +select * from mysql.column_stats; +--echo # filtered shows that we used the data from stat tables +analyze +select * from t1 where a = 1 and b=3; + +--echo # +--echo # with use_stat_tables= PREFERABLY +--echo # analyze table t1 will collect statistics +--echo # + +set @@use_stat_tables=PREFERABLY; +analyze table t1; +select * from mysql.column_stats; +--echo # filtered shows that we used the data from stat tables +analyze +select * from t1 where a=1 and b=3; +drop table t1; + set use_stat_tables=@save_use_stat_tables; diff --git a/mysql-test/main/stat_tables_innodb.result b/mysql-test/main/stat_tables_innodb.result index 8198e94dc10..4bb8f34712c 100644 --- a/mysql-test/main/stat_tables_innodb.result +++ b/mysql-test/main/stat_tables_innodb.result @@ -632,6 +632,65 @@ SELECT MAX(pk) FROM t1; MAX(pk) NULL DROP TABLE t1; +# +# MDEV-17255: New optimizer defaults and ANALYZE TABLE +# +set @save_use_stat_tables= @@use_stat_tables; +create table t1 (a int, b int); +insert into t1(a,b) values (1,2),(1,3),(1,4),(1,5),(2,6),(2,7),(3,8),(3,9),(3,9),(4,10); +# +# with use_stat_tables= PREFERABLY_FOR_QUERIES +# analyze table t1 will not collect statistics +# +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +select * from mysql.column_stats; +db_name table_name column_name min_value max_value nulls_ratio avg_length avg_frequency hist_size hist_type histogram +test t1 a 1 4 0.0000 4.0000 2.5000 0 NULL NULL +test t1 b 2 10 0.0000 4.0000 1.1111 0 NULL NULL +analyze +select * from t1 where a = 1 and b=3; +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 10 10.00 2.78 10.00 Using where +# +# with use_stat_tables= PREFERABLY_FOR_QUERIES +# analyze table t1 will collect statistics if we use PERSISTENT +# for columns, indexes or everything +# +analyze table t1 persistent for columns (a) indexes (); +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +select * from mysql.column_stats; +db_name table_name column_name min_value max_value nulls_ratio avg_length avg_frequency hist_size hist_type histogram +test t1 a 1 4 0.0000 4.0000 2.5000 0 NULL NULL +test t1 b 2 10 0.0000 4.0000 1.1111 0 NULL NULL +# filtered shows that we used the data from stat tables +analyze +select * from t1 where a = 1 and b=3; +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 10 10.00 2.78 10.00 Using where +# +# with use_stat_tables= PREFERABLY +# analyze table t1 will collect statistics +# +set @@use_stat_tables=PREFERABLY; +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +select * from mysql.column_stats; +db_name table_name column_name min_value max_value nulls_ratio avg_length avg_frequency hist_size hist_type histogram +test t1 a 1 4 0.0000 4.0000 2.5000 0 NULL NULL +test t1 b 2 10 0.0000 4.0000 1.1111 0 NULL NULL +# filtered shows that we used the data from stat tables +analyze +select * from t1 where a=1 and b=3; +id select_type table type possible_keys key key_len ref rows r_rows filtered r_filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 10 10.00 2.78 10.00 Using where +drop table t1; set use_stat_tables=@save_use_stat_tables; set optimizer_switch=@save_optimizer_switch_for_stat_tables_test; SET SESSION STORAGE_ENGINE=DEFAULT; diff --git a/sql/sql_admin.cc b/sql/sql_admin.cc index d0d959de8f9..b39103e382a 100644 --- a/sql/sql_admin.cc +++ b/sql/sql_admin.cc @@ -767,7 +767,7 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, } collect_eis= (table->table->s->table_category == TABLE_CATEGORY_USER && - (get_use_stat_tables_mode(thd) > NEVER || + (check_eits_collection_allowed(thd) || lex->with_persistent_for_clause)); diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc index 04806f07b3b..8c88f7f927f 100644 --- a/sql/sql_statistics.cc +++ b/sql/sql_statistics.cc @@ -3720,9 +3720,8 @@ void set_statistics_for_table(THD *thd, TABLE *table) { TABLE_STATISTICS_CB *stats_cb= &table->s->stats_cb; Table_statistics *read_stats= stats_cb->table_stats; - Use_stat_tables_mode use_stat_table_mode= get_use_stat_tables_mode(thd); table->used_stat_records= - (use_stat_table_mode <= COMPLEMENTARY || + (!check_eits_preferred(thd) || !table->stats_is_read || read_stats->cardinality_is_null) ? table->file->stats.records : read_stats->cardinality; KEY *key_info, *key_info_end; @@ -3730,7 +3729,7 @@ void set_statistics_for_table(THD *thd, TABLE *table) key_info < key_info_end; key_info++) { key_info->is_statistics_from_stat_tables= - (use_stat_table_mode > COMPLEMENTARY && + (check_eits_preferred(thd) && table->stats_is_read && key_info->read_stats->avg_frequency_is_inited() && key_info->read_stats->get_avg_frequency(0) > 0.5); diff --git a/sql/sql_statistics.h b/sql/sql_statistics.h index 39cddf95188..6b08622e8e4 100644 --- a/sql/sql_statistics.h +++ b/sql/sql_statistics.h @@ -22,6 +22,8 @@ enum enum_use_stat_tables_mode NEVER, COMPLEMENTARY, PREFERABLY, + COMPLEMENTARY_FOR_QUERIES, + PREFERABLY_FOR_QUERIES } Use_stat_tables_mode; typedef @@ -87,6 +89,19 @@ Use_stat_tables_mode get_use_stat_tables_mode(THD *thd) { return (Use_stat_tables_mode) (thd->variables.use_stat_tables); } +inline +bool check_eits_collection_allowed(THD *thd) +{ + return (get_use_stat_tables_mode(thd) == COMPLEMENTARY || + get_use_stat_tables_mode(thd) == PREFERABLY); +} + +inline +bool check_eits_preferred(THD *thd) +{ + return (get_use_stat_tables_mode(thd) == PREFERABLY || + get_use_stat_tables_mode(thd) == PREFERABLY_FOR_QUERIES); +} int read_statistics_for_tables_if_needed(THD *thd, TABLE_LIST *tables); int collect_statistics_for_table(THD *thd, TABLE *table); diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 420e7feabde..b925de34db4 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -5842,12 +5842,13 @@ static Sys_var_ulong Sys_progress_report_time( VALID_RANGE(0, UINT_MAX), DEFAULT(5), BLOCK_SIZE(1)); const char *use_stat_tables_modes[] = - {"NEVER", "COMPLEMENTARY", "PREFERABLY", 0}; + {"NEVER", "COMPLEMENTARY", "PREFERABLY", + "COMPLEMENTARY_FOR_QUERIES", "PREFERABLY_FOR_QUERIES", 0}; static Sys_var_enum Sys_optimizer_use_stat_tables( "use_stat_tables", "Specifies how to use system statistics tables", SESSION_VAR(use_stat_tables), CMD_LINE(REQUIRED_ARG), - use_stat_tables_modes, DEFAULT(2)); + use_stat_tables_modes, DEFAULT(4)); static Sys_var_ulong Sys_histogram_size( "histogram_size",

1 0

[Commits] 4a9f2e73d14: MDEV-16188: Rowid filters
by Sergei Petrunia 04 Dec '18

04 Dec '18

revision-id: 4a9f2e73d140ae8cb7c52486305cf00641c378e4 (mariadb-10.3.6-99-g4a9f2e73d14) parent(s): 90708fc15cf835d03b511a953ae6939081a0f9e1 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2018-12-05 01:43:31 +0300 message: MDEV-16188: Rowid filters Add support for EXPLAIN FORMAT=JSON. --- mysql-test/main/rowid_filter.result | 50 +++++++++++++++++++++++++- mysql-test/main/rowid_filter.test | 6 ++++ sql/sql_explain.cc | 72 +++++++++++++++++++++---------------- sql/sql_explain.h | 44 +++++++++++++++++------ sql/sql_select.cc | 36 +++++-------------- sql/sql_select.h | 1 - 6 files changed, 138 insertions(+), 71 deletions(-) diff --git a/mysql-test/main/rowid_filter.result b/mysql-test/main/rowid_filter.result index e1296c35f88..c5267c1a6ed 100644 --- a/mysql-test/main/rowid_filter.result +++ b/mysql-test/main/rowid_filter.result @@ -146,6 +146,54 @@ FROM orders JOIN lineitem ON o_orderkey=l_orderkey WHERE l_shipdate BETWEEN '1997-01-09' AND '1997-01-10' AND l_receiptdate BETWEEN '1997-01-09' AND '1997-01-10'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE lineitem range|filter PRIMARY,i_l_shipdate,i_l_receiptdate,i_l_orderkey,i_l_orderkey_quantity i_l_receiptdate|i_l_shipdate 4|4 NULL 6 (1%) Using index condition; Using where; Using filter +1 SIMPLE lineitem range|filter PRIMARY,i_l_shipdate,i_l_receiptdate,i_l_orderkey,i_l_orderkey_quantity i_l_receiptdate|i_l_shipdate 4|4 NULL 6 (0%) Using index condition; Using where; Using filter 1 SIMPLE orders eq_ref PRIMARY PRIMARY 4 dbt3_s001.lineitem.l_orderkey 1 +# Same as above but EXPLAIN FORMAT=JSON: +EXPLAIN FORMAT=JSON SELECT * +FROM orders JOIN lineitem ON o_orderkey=l_orderkey +WHERE l_shipdate BETWEEN '1997-01-09' AND '1997-01-10' AND +l_receiptdate BETWEEN '1997-01-09' AND '1997-01-10'; +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "lineitem", + "access_type": "range", + "possible_keys": [ + "PRIMARY", + "i_l_shipdate", + "i_l_receiptdate", + "i_l_orderkey", + "i_l_orderkey_quantity" + ], + "key": "i_l_receiptdate", + "key_length": "4", + "used_key_parts": ["l_receiptDATE"], + "rowid_filter": { + "range": { + "key": "i_l_shipdate", + "used_key_parts": ["l_shipDATE"] + }, + "rows": 8, + "selectivity_pct": 0.1332 + }, + "rows": 6, + "filtered": 100, + "index_condition": "lineitem.l_receiptDATE between '1997-01-09' and '1997-01-10'", + "attached_condition": "lineitem.l_shipDATE between '1997-01-09' and '1997-01-10'" + }, + "table": { + "table_name": "orders", + "access_type": "eq_ref", + "possible_keys": ["PRIMARY"], + "key": "PRIMARY", + "key_length": "4", + "used_key_parts": ["o_orderkey"], + "ref": ["dbt3_s001.lineitem.l_orderkey"], + "rows": 1, + "filtered": 100 + } + } +} DROP DATABASE dbt3_s001; diff --git a/mysql-test/main/rowid_filter.test b/mysql-test/main/rowid_filter.test index 4f41c408d94..73d7326623c 100644 --- a/mysql-test/main/rowid_filter.test +++ b/mysql-test/main/rowid_filter.test @@ -139,4 +139,10 @@ FROM orders JOIN lineitem ON o_orderkey=l_orderkey WHERE l_shipdate BETWEEN '1997-01-09' AND '1997-01-10' AND l_receiptdate BETWEEN '1997-01-09' AND '1997-01-10'; +--echo # Same as above but EXPLAIN FORMAT=JSON: +EXPLAIN FORMAT=JSON SELECT * +FROM orders JOIN lineitem ON o_orderkey=l_orderkey +WHERE l_shipdate BETWEEN '1997-01-09' AND '1997-01-10' AND + l_receiptdate BETWEEN '1997-01-09' AND '1997-01-10'; + DROP DATABASE dbt3_s001; diff --git a/sql/sql_explain.cc b/sql/sql_explain.cc index c7e9d6c906b..f7262e2624b 100644 --- a/sql/sql_explain.cc +++ b/sql/sql_explain.cc @@ -1115,12 +1115,14 @@ void Explain_table_access::fill_key_str(String *key_str, bool is_json) const - this is just used key length for ref/range - for index_merge, it is a comma-separated list of lengths. - for hash join, it is key_len:pseudo_key_len + - [tabular form only] rowid filter length is added after "|". - The column looks identical in tabular and json forms. In JSON, we consider - the column legacy, it is superceded by used_key_parts. + In JSON, we consider this column to be legacy, it is superceded by + used_key_parts. */ -void Explain_table_access::fill_key_len_str(String *key_len_str) const +void Explain_table_access::fill_key_len_str(String *key_len_str, + bool is_json) const { bool is_hj= (type == JT_HASH || type == JT_HASH_NEXT || type == JT_HASH_RANGE || type == JT_HASH_INDEX_MERGE); @@ -1149,13 +1151,12 @@ void Explain_table_access::fill_key_len_str(String *key_len_str) const key_len_str->append(buf, length); } - if (key.get_filter_len() != (uint)-1) + if (!is_json && rowid_filter) { - char buf[64]; - size_t length; key_len_str->append('|'); - length= longlong10_to_str(key.get_filter_len(), buf, 10) - buf; - key_len_str->append(buf, length); + StringBuffer<64> filter_key_len; + rowid_filter->quick->print_key_len(&filter_key_len); + key_len_str->append(filter_key_len); } } @@ -1179,20 +1180,6 @@ bool Explain_index_use::set(MEM_ROOT *mem_root, KEY *key, uint key_len_arg) return 0; } -bool Explain_index_use::set_filter(MEM_ROOT *mem_root, KEY *key, uint key_len_arg) -{ - if (!(filter_name= strdup_root(mem_root, key->name.str))) - return 1; - filter_len= key_len_arg; - uint len= 0; - for (uint i= 0; i < key->usable_key_parts; i++) - { - len += key->key_part[i].store_length; - if (len >= key_len_arg) - break; - } - return 0; -} bool Explain_index_use::set_pseudo_key(MEM_ROOT *root, const char* key_name_arg) { @@ -1257,14 +1244,16 @@ int Explain_table_access::print_explain(select_result_sink *output, uint8 explai /* `type` column */ StringBuffer<64> join_type_buf; - if (!is_filter_set()) + if (rowid_filter == NULL) push_str(thd, &item_list, join_type_str[type]); else { join_type_buf.append(join_type_str[type]); join_type_buf.append("|filter"); item_list.push_back(new (mem_root) - Item_string_sys(thd, join_type_buf.ptr(), join_type_buf.length()), mem_root); + Item_string_sys(thd, join_type_buf.ptr(), + join_type_buf.length()), + mem_root); } /* `possible_keys` column */ @@ -1277,10 +1266,13 @@ int Explain_table_access::print_explain(select_result_sink *output, uint8 explai /* `key` */ StringBuffer<64> key_str; fill_key_str(&key_str, false); - if (key.get_filter_name()) + + if (rowid_filter) { key_str.append("|"); - key_str.append(key.get_filter_name()); + StringBuffer<64> rowid_key_str; + rowid_filter->quick->print_key(&rowid_key_str); + key_str.append(rowid_key_str); } if (key_str.length() > 0) @@ -1290,7 +1282,7 @@ int Explain_table_access::print_explain(select_result_sink *output, uint8 explai /* `key_len` */ StringBuffer<64> key_len_str; - fill_key_len_str(&key_len_str); + fill_key_len_str(&key_len_str, false); if (key_len_str.length() > 0) push_string(thd, &item_list, &key_len_str); @@ -1318,10 +1310,10 @@ int Explain_table_access::print_explain(select_result_sink *output, uint8 explai { rows_str.append_ulonglong((ulonglong)rows); - if (is_filter_set()) + if (rowid_filter) { rows_str.append(" ("); - rows_str.append_ulonglong(filter_perc); + rows_str.append_ulonglong(round(rowid_filter->selectivity * 100.0)); rows_str.append("%)"); } item_list.push_back(new (mem_root) @@ -1406,7 +1398,7 @@ int Explain_table_access::print_explain(select_result_sink *output, uint8 explai extra_buf.append(STRING_WITH_LEN("Using filesort")); } - if (is_filter_set()) + if (rowid_filter) { if (first) first= false; @@ -1603,6 +1595,19 @@ void add_json_keyset(Json_writer *writer, const char *elem_name, } +void Explain_rowid_filter::print_explain_json(Explain_query *query, + Json_writer *writer, + bool is_analyze) +{ + Json_writer_nesting_guard guard(writer); + writer->add_member("rowid_filter").start_object(); + quick->print_json(writer); + writer->add_member("rows").add_ll(rows); + writer->add_member("selectivity_pct").add_double(selectivity * 100.0); + writer->end_object(); // rowid_filter +} + + void Explain_table_access::print_explain_json(Explain_query *query, Json_writer *writer, bool is_analyze) @@ -1675,7 +1680,7 @@ void Explain_table_access::print_explain_json(Explain_query *query, /* `key_length` */ StringBuffer<64> key_len_str; - fill_key_len_str(&key_len_str); + fill_key_len_str(&key_len_str, true); if (key_len_str.length()) writer->add_member("key_length").add_str(key_len_str); @@ -1700,6 +1705,11 @@ void Explain_table_access::print_explain_json(Explain_query *query, if (!ref_list.is_empty()) print_json_array(writer, "ref", ref_list); + if (rowid_filter) + { + rowid_filter->print_explain_json(query, writer, is_analyze); + } + /* r_loops (not present in tabular output) */ if (is_analyze) { diff --git a/sql/sql_explain.h b/sql/sql_explain.h index fc4e3462162..89fbd0f757d 100644 --- a/sql/sql_explain.h +++ b/sql/sql_explain.h @@ -602,12 +602,38 @@ class Explain_index_use : public Sql_alloc } bool set(MEM_ROOT *root, KEY *key_name, uint key_len_arg); bool set_pseudo_key(MEM_ROOT *root, const char *key_name); - bool set_filter(MEM_ROOT *root, KEY *key, uint key_len_arg); inline const char *get_key_name() const { return key_name; } inline uint get_key_len() const { return key_len; } - inline const char *get_filter_name() const { return filter_name; } - inline uint get_filter_len() const { return filter_len; } + //inline const char *get_filter_name() const { return filter_name; } +}; + + +/* + Query Plan data structure for Rowid filter. +*/ +class Explain_rowid_filter : public Sql_alloc +{ +public: + /* Quick select used to collect the rowids into filter */ + Explain_quick_select *quick; + + /* How many rows the above quick select is expected to return */ + ha_rows rows; + + /* Expected selectivity for the filter */ + double selectivity; + + void print_explain_json(Explain_query *query, Json_writer *writer, + bool is_analyze); + + /* + TODO: + Here should be ANALYZE members: + - r_rows for the quick select + - An object that tracked the table access time + - real selectivity of the filter. + */ }; @@ -677,6 +703,7 @@ class Explain_range_checked_fer : public Sql_alloc void print_json(Json_writer *writer, bool is_analyze); }; + /* EXPLAIN data structure for a single JOIN_TAB. */ @@ -697,7 +724,7 @@ class Explain_table_access : public Sql_alloc pushed_index_cond(NULL), sjm_nest(NULL), pre_join_sort(NULL), - filter_perc(UINT_MAX) + rowid_filter(NULL) {} ~Explain_table_access() { delete sjm_nest; } @@ -804,12 +831,7 @@ class Explain_table_access : public Sql_alloc Exec_time_tracker op_tracker; Table_access_tracker jbuf_tracker; - /** - How many rows are left after the filter was applied - to the initial rows count in percentages. - */ - double filter_perc; - inline bool is_filter_set() const { return (filter_perc != UINT_MAX); } + Explain_rowid_filter *rowid_filter; int print_explain(select_result_sink *output, uint8 explain_flags, bool is_analyze, @@ -821,7 +843,7 @@ class Explain_table_access : public Sql_alloc private: void append_tag_name(String *str, enum explain_extra_tag tag); void fill_key_str(String *key_str, bool is_json) const; - void fill_key_len_str(String *key_len_str) const; + void fill_key_len_str(String *key_len_str, bool is_json) const; double get_r_filtered(); void tag_to_json(Json_writer *writer, enum explain_extra_tag tag); }; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index c194fdb8824..2cb357c1eff 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -25167,21 +25167,6 @@ int append_possible_keys(MEM_ROOT *alloc, String_list &list, TABLE *table, } -/** - This method saves the data that should be printed in EXPLAIN - if any filter was used for this table. -*/ - -bool JOIN_TAB::save_filter_explain_data(Explain_table_access *eta) -{ - if (!filter) - return 0; - (filter->selectivity*100 >= 1) ? eta->filter_perc= round(filter->selectivity*100) : - eta->filter_perc= 1; - return 0; -} - - bool JOIN_TAB::save_explain_data(Explain_table_access *eta, table_map prefix_tables, bool distinct_arg, JOIN_TAB *first_top_tab) @@ -25217,7 +25202,7 @@ bool JOIN_TAB::save_explain_data(Explain_table_access *eta, filesort))) return 1; } - + // psergey-todo: data for filtering! tracker= &eta->tracker; jbuf_tracker= &eta->jbuf_tracker; @@ -25318,12 +25303,16 @@ bool JOIN_TAB::save_explain_data(Explain_table_access *eta, // psergey-todo: ^ check for error return code /* Build "key", "key_len", and "ref" */ - if (filter) { - eta->key.set_filter(thd->mem_root, - &filter->table->key_info[filter->key_no], - rowid_filter->get_container()->get_select()->quick->max_used_key_length); + QUICK_SELECT_I *quick= rowid_filter->get_container()->get_select()->quick; + + Explain_rowid_filter *erf= new (thd->mem_root) Explain_rowid_filter; + erf->quick= quick->get_explain(thd->mem_root); + erf->selectivity= filter->selectivity; + erf->rows= quick->records; + eta->rowid_filter= erf; + //psergey-todo: also do setup for ANALYZE here. } if (tab_type == JT_NEXT) @@ -25445,13 +25434,6 @@ bool JOIN_TAB::save_explain_data(Explain_table_access *eta, eta->filtered= f; } - if ((tab_select && tab_select->quick && tab_type != JT_CONST) || - (key_info && ref.key_parts && tab_type != JT_FT)) - { - if (save_filter_explain_data(eta)) - return 1; - } - /* Build "Extra" field and save it */ key_read= table->file->keyread_enabled(); if ((tab_type == JT_NEXT || tab_type == JT_CONST) && diff --git a/sql/sql_select.h b/sql/sql_select.h index 1df7e5e9f5b..2fea84d0bab 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -664,7 +664,6 @@ typedef struct st_join_table { SplM_plan_info *choose_best_splitting(double record_count, table_map remaining_tables); bool fix_splitting(SplM_plan_info *spl_plan, table_map remaining_tables); - bool save_filter_explain_data(Explain_table_access *eta); } JOIN_TAB;

1 0

[Commits] 8400b2c356d: Make Range Locking support IS.rocksdb_trx
by Sergei Petrunia 03 Dec '18

03 Dec '18

revision-id: 8400b2c356d07d91ec49c559d859ab6f4798c789 (fb-prod201801-174-g8400b2c356d) parent(s): 9ae6a29c90a1d6d1c7eaa2e642dd3e876957d0a2 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2018-12-03 19:47:48 +0300 message: Make Range Locking support IS.rocksdb_trx - Make lock_request functions that accept lock_wait_callback also accept callback_arg argument to pass to the callback (it used to just pass nullptr). - RocksDB update uses that callback to make RocksDB transaction "know" which rowkey it is waiting on. - Don't run rocksdb.i_s_deadlock in range locking mode as it uses SELECT LOCK IN SHARE MODE --- mysql-test/suite/rocksdb/t/i_s_deadlock.test | 4 ++++ rocksdb | 2 +- storage/rocksdb/range_locking/locktree/lock_request.cc | 14 +++++++++----- storage/rocksdb/range_locking/locktree/lock_request.h | 7 +++++-- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/mysql-test/suite/rocksdb/t/i_s_deadlock.test b/mysql-test/suite/rocksdb/t/i_s_deadlock.test index e0479d6a337..82fa9fc6bbd 100644 --- a/mysql-test/suite/rocksdb/t/i_s_deadlock.test +++ b/mysql-test/suite/rocksdb/t/i_s_deadlock.test @@ -1,5 +1,9 @@ --source include/have_rocksdb.inc +# Uses LOCK IN SHARE MODE and so will hang in range-locking mode. The part that +# doesn't hang is in rocksdb.range_locking_i_s_deadlock.test +--source suite/rocksdb/include/not_range_locking.inc + set @prior_lock_wait_timeout = @@rocksdb_lock_wait_timeout; set @prior_deadlock_detect = @@rocksdb_deadlock_detect; set @prior_max_latest_deadlocks = @@rocksdb_max_latest_deadlocks; diff --git a/rocksdb b/rocksdb index 574dd4e574d..7f29e9512f6 160000 --- a/rocksdb +++ b/rocksdb @@ -1 +1 @@ -Subproject commit 574dd4e574d60263db9d37faede89057caefafca +Subproject commit 7f29e9512f6bd7a12d8a8d3a493eaff586c5f28b diff --git a/storage/rocksdb/range_locking/locktree/lock_request.cc b/storage/rocksdb/range_locking/locktree/lock_request.cc index 2e20bcf5249..307c57d9aac 100644 --- a/storage/rocksdb/range_locking/locktree/lock_request.cc +++ b/storage/rocksdb/range_locking/locktree/lock_request.cc @@ -202,7 +202,8 @@ int lock_request::wait(uint64_t wait_time_ms) { } int lock_request::wait(uint64_t wait_time_ms, uint64_t killed_time_ms, int (*killed_callback)(void), - void (*lock_wait_callback)(void *, TXNID, TXNID)) { + void (*lock_wait_callback)(void *, TXNID, TXNID), + void *callback_arg) { uint64_t t_now = toku_current_time_microsec(); uint64_t t_start = t_now; uint64_t t_end = t_start + wait_time_ms * 1000; @@ -215,7 +216,8 @@ int lock_request::wait(uint64_t wait_time_ms, uint64_t killed_time_ms, int (*kil conflicts_collector.init(); retry(&conflicts_collector); if (m_state == state::PENDING) { - report_waits(&conflicts_collector, lock_wait_callback); + report_waits(&conflicts_collector, lock_wait_callback, + callback_arg); } conflicts_collector.deinit(); } @@ -330,6 +332,7 @@ int lock_request::retry(GrowableArray<TXNID> *conflicts_collector) { void lock_request::retry_all_lock_requests( locktree *lt, void (*lock_wait_callback)(void *, TXNID, TXNID), + void *callback_arg, void (*after_retry_all_test_callback)(void)) { lt_lock_request_info *info = lt->get_lock_request_info(); @@ -372,7 +375,7 @@ void lock_request::retry_all_lock_requests( } toku_mutex_unlock(&info->retry_mutex); - report_waits(&conflicts_collector, lock_wait_callback); + report_waits(&conflicts_collector, lock_wait_callback, callback_arg); conflicts_collector.deinit(); } @@ -409,14 +412,15 @@ void lock_request::add_conflicts_to_waits(txnid_set *conflicts, } void lock_request::report_waits(GrowableArray<TXNID> *wait_conflicts, - void (*lock_wait_callback)(void *, TXNID, TXNID)) { + void (*lock_wait_callback)(void *, TXNID, TXNID), + void *callback_arg) { if (!lock_wait_callback) return; size_t num_conflicts = wait_conflicts->get_size(); for (size_t i = 0; i < num_conflicts; i += 2) { TXNID blocked_txnid = wait_conflicts->fetch_unchecked(i); TXNID blocking_txnid = wait_conflicts->fetch_unchecked(i+1); - (*lock_wait_callback)(nullptr, blocked_txnid, blocking_txnid); + (*lock_wait_callback)(callback_arg, blocked_txnid, blocking_txnid); } } diff --git a/storage/rocksdb/range_locking/locktree/lock_request.h b/storage/rocksdb/range_locking/locktree/lock_request.h index 9e82b31541e..4fe88e9162f 100644 --- a/storage/rocksdb/range_locking/locktree/lock_request.h +++ b/storage/rocksdb/range_locking/locktree/lock_request.h @@ -90,7 +90,8 @@ public: // or simply DB_LOCK_NOTGRANTED if the wait time expired. int wait(uint64_t wait_time_ms); int wait(uint64_t wait_time_ms, uint64_t killed_time_ms, int (*killed_callback)(void), - void (*lock_wait_callback)(void *, TXNID, TXNID) = nullptr); + void (*lock_wait_callback)(void *, TXNID, TXNID) = nullptr, + void *callback_arg= nullptr); // return: left end-point of the lock range const DBT *get_left_key(void) const; @@ -114,6 +115,7 @@ public: static void retry_all_lock_requests( locktree *lt, void (*lock_wait_callback)(void *, TXNID, TXNID) = nullptr, + void *callback_arg= nullptr, void (*after_retry_test_callback)(void) = nullptr); static void retry_all_lock_requests_info(lt_lock_request_info *info, GrowableArray<TXNID> *collector); @@ -202,7 +204,8 @@ public: // Report list of conflicts to lock wait callback. static void report_waits(GrowableArray<TXNID> *wait_conflicts, - void (*lock_wait_callback)(void *, TXNID, TXNID)); + void (*lock_wait_callback)(void *, TXNID, TXNID), + void *callback_arg); void add_conflicts_to_waits(txnid_set *conflicts, GrowableArray<TXNID> *wait_conflicts); void (*m_start_test_callback)(void);

1 0

[Commits] 7f29e9512: Range Locking: make RangeLockMgr call SetWaitingTxn/ClearWaitingTxn
by Sergei Petrunia 03 Dec '18

03 Dec '18

revision-id: 7f29e9512f6bd7a12d8a8d3a493eaff586c5f28b (v5.8-1020-g7f29e9512) parent(s): 574dd4e574d60263db9d37faede89057caefafca author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2018-12-03 19:37:47 +0300 message: Range Locking: make RangeLockMgr call SetWaitingTxn/ClearWaitingTxn In a similar way to how TransactionLockMgr does it. --- utilities/transactions/transaction_lock_mgr.cc | 45 ++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/utilities/transactions/transaction_lock_mgr.cc b/utilities/transactions/transaction_lock_mgr.cc index e43d17fd1..a7eb7e30b 100644 --- a/utilities/transactions/transaction_lock_mgr.cc +++ b/utilities/transactions/transaction_lock_mgr.cc @@ -321,6 +321,7 @@ void RangeLockMgr::KillLockWait(void *cdata) } + // Get a range lock on [start_key; end_key] range // (TODO: check if we do what is inteded at the endpoints) @@ -346,9 +347,47 @@ Status RangeLockMgr::TryRangeLock(PessimisticTransaction* txn, wait_time_msec = (wait_time_msec + 500) / 1000; request.start(); - const int r = request.wait(wait_time_msec, killed_time_msec, - nullptr, /* killed_callback */ - nullptr /* lock_wait_needed_callback*/ ); + + /* + If we are going to wait on the lock, we should set appropriate status in + the 'txn' object. This is done by the SetWaitingTxn() call below. + The API we are using are MariaDB's wait notification API, so the way this + is done is a bit convoluted. + In MyRocks, the wait details are visible in I_S.rocksdb_trx. + */ + std::string key_str(start_key.data(), start_key.size()); + struct st_wait_info { + PessimisticTransaction* txn; + uint32_t column_family_id; + std::string *key_ptr; + autovector<TransactionID> wait_ids; + bool done= false; + + static void lock_wait_callback(void *cdata, TXNID waiter, TXNID waitee) + { + auto self= (struct st_wait_info*)cdata; + if (!self->done) + { + self->wait_ids.push_back(waitee); + self->txn->SetWaitingTxn(self->wait_ids, self->column_family_id, self->key_ptr); + self->done= true; + } + } + } wait_info; + + wait_info.txn= txn; + wait_info.column_family_id= column_family_id; + wait_info.key_ptr= &key_str; + wait_info.done= false; + + const int r = request.wait(wait_time_msec, killed_time_msec, + nullptr, // killed_callback + st_wait_info::lock_wait_callback, + (void*)&wait_info); + + // Inform the txn that we are no longer waiting: + txn->ClearWaitingTxn(); + request.destroy(); switch (r) { case 0:

1 0

[Commits] af735cd52e5: Postreview part 1 (v1)
by Oleksandr Byelkin 03 Dec '18

03 Dec '18

revision-id: af735cd52e54959dedc542acd23b89551cccf19b (mariadb-10.4.0-39-gaf735cd52e5) parent(s): 2fef5d8d8e5b5781a71ff6305c0eb7a3de555509 author: Oleksandr Byelkin committer: Oleksandr Byelkin timestamp: 2018-12-03 15:05:49 +0100 message: Postreview part 1 (v1) --- sql/protocol.h | 1 + sql/sql_class.cc | 154 +++++++++++++++++++++++++------------------------------ sql/sql_class.h | 7 +-- 3 files changed, 75 insertions(+), 87 deletions(-) diff --git a/sql/protocol.h b/sql/protocol.h index bf74f52fa98..2b9f1a592e4 100644 --- a/sql/protocol.h +++ b/sql/protocol.h @@ -34,6 +34,7 @@ struct insert_id_desc { ulonglong first_id; ulonglong sequence; + ulong collect_auto_increment_increment; }; class Protocol diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 87b83e60bd9..1f8f011bfa6 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -638,9 +638,8 @@ THD::THD(my_thread_id id, bool is_wsrep_applier, bool skip_global_sys_var_lock) m_stmt_da(&main_da), tdc_hash_pins(0), xid_hash_pins(0), - m_tmp_tables_locked(false) + m_tmp_tables_locked(false), #ifdef WITH_WSREP - , wsrep_applier(is_wsrep_applier), wsrep_applier_closing(false), wsrep_client_thread(false), @@ -648,8 +647,9 @@ THD::THD(my_thread_id id, bool is_wsrep_applier, bool skip_global_sys_var_lock) wsrep_po_handle(WSREP_PO_INITIALIZER), wsrep_po_cnt(0), wsrep_apply_format(0), - wsrep_ignore_table(false) + wsrep_ignore_table(false), #endif + insert_id_info(NULL) { ulong tmp; bzero(&variables, sizeof(variables)); @@ -1261,7 +1261,7 @@ void THD::init(bool skip_lock) #endif //EMBEDDED_LIBRARY apc_target.init(&LOCK_thd_kill); - insert_ids= NULL; + insert_id_info= NULL; DBUG_VOID_RETURN; } @@ -7695,113 +7695,99 @@ bool Qualified_column_ident::append_to(THD *thd, String *str) const bool THD::init_collecting_insert_id() { - if (!insert_ids) - { - void *buff; - if (!(my_multi_malloc(MYF(MY_WME), &insert_ids, sizeof(DYNAMIC_ARRAY), - &buff, sizeof(insert_id_desc) * 10, - NullS)) || - my_init_dynamic_array2(insert_ids, sizeof(insert_id_desc), - buff, 10, 100, MYF(MY_WME))) - { - if (insert_ids) - my_free(insert_ids); - insert_ids= NULL; + List<Item> field_list; + + // it should not be done on statement memory) + DBUG_ASSERT(stmt_arena->is_conventional() || + mem_root != stmt_arena->mem_root); + + DBUG_ASSERT(insert_id_info == NULL); + if ((insert_id_info= (insert_id_desc*) calloc(sizeof(insert_id_desc)))) + return TRUE; + + field_list.push_back(new (mem_root) + Item_int(this, "Id", 0, MY_INT64_NUM_DECIMAL_DIGITS), + mem_root); + field_list.push_back(new (mem_root) + Item_int(this, "Len", 0, MY_INT64_NUM_DECIMAL_DIGITS), + mem_root); + field_list.push_back(new (mem_root) + Item_return_int(this, "Inc", 0, MYSQL_TYPE_LONG), + mem_root); + + if (protocol_binary.send_result_set_metadata(&field_list, + Protocol::SEND_NUM_ROWS | + Protocol::SEND_EOF)) return TRUE; - } - collect_auto_increment_increment= variables.auto_increment_increment; - } + + return FALSE; } -void THD::stop_collecting_insert_id() + +bool THD::send_collected_insert_id() { - if (insert_ids) - { - delete_dynamic(insert_ids); - my_free(insert_ids); - insert_ids= NULL; - } + DBUG_ASSERT(insert_id_info->collect_auto_increment_increment); + protocol_binary.prepare_for_resend(); + protocol_binary.store_longlong(insert_id_info->first_id, TRUE); + protocol_binary.store_longlong(insert_id_info->sequence, TRUE); + protocol_binary.store_long(insert_id_info-> + collect_auto_increment_increment); + if (protocol_binary.write()) + return TRUE; + return FALSE; } + bool THD::collect_insert_id(ulonglong id) { - if (insert_ids) + if (insert_id_info) { - if (insert_ids->elements) + if (insert_id_info->collect_auto_increment_increment) { - insert_id_desc *last= - (insert_id_desc *)dynamic_array_ptr(insert_ids, - insert_ids->elements - 1); - if (id == last->first_id) + if (id == insert_id_info->first_id) { return FALSE; // no new insert id } - if (id == last->first_id + (last->sequence * - collect_auto_increment_increment)) + if (id == insert_id_info->first_id + (insert_id_info->sequence * + insert_id_info-> + collect_auto_increment_increment)) { - last->sequence++; + insert_id_info->sequence++; return FALSE; } + + if (send_collected_insert_id()) + return TRUE; } - insert_id_desc el; - el.first_id= id; - el.sequence= 1; - if (insert_dynamic(insert_ids, &el)) - { - return TRUE; - } + else + insert_id_info->collect_auto_increment_increment= + variables.auto_increment_increment; + + insert_id_info->first_id= id; + insert_id_info->sequence= 1; } return FALSE; } +/** + Check is insert ID data was connected and push out rest of it and stop + collecting + + @retval TRUE insert IDs was colleted + @retval FALSE insert IDs was not colleted +*/ bool THD::report_collected_insert_id() { - if (insert_ids) + if (insert_id_info) { - List<Item> field_list; - MEM_ROOT tmp_mem_root; - Query_arena arena(&tmp_mem_root, Query_arena::STMT_INITIALIZED), backup; - - init_alloc_root(arena.mem_root, " THD::report_collected_insert_i", - 2048, 4096, MYF(MY_THREAD_SPECIFIC)); - set_n_backup_active_arena(&arena, &backup); - DBUG_ASSERT(mem_root == &tmp_mem_root); - - field_list.push_back(new (mem_root) - Item_int(this, "Id", 0, MY_INT64_NUM_DECIMAL_DIGITS), - mem_root); - field_list.push_back(new (mem_root) - Item_int(this, "Len", 0, MY_INT64_NUM_DECIMAL_DIGITS), - mem_root); - field_list.push_back(new (mem_root) - Item_return_int(this, "Inc", 0, MYSQL_TYPE_LONG), - mem_root); - - if (protocol_binary.send_result_set_metadata(&field_list, - Protocol::SEND_NUM_ROWS | Protocol::SEND_EOF)) - goto error; + /* + No need to check an error it will be done on after exit from this + method + */ + send_collected_insert_id(); - for (ulonglong i= 0; i < insert_ids->elements; i++) - { - insert_id_desc *last= - (insert_id_desc *)dynamic_array_ptr(insert_ids, i); - if (insert_ids->elements == 1 && last->first_id == 0 && - get_stmt_da()->affected_rows() != 1) - continue; // No insert IDs - protocol_binary.prepare_for_resend(); - protocol_binary.store_longlong(last->first_id, TRUE); - protocol_binary.store_longlong(last->sequence, TRUE); - protocol_binary.store_long(collect_auto_increment_increment); - if (protocol_binary.write()) - goto error; - } -error: - restore_active_arena(&arena, &backup); - DBUG_ASSERT(arena.mem_root == &tmp_mem_root); - // no need free Items because they was only constants - free_root(arena.mem_root, MYF(0)); stop_collecting_insert_id(); return TRUE; } diff --git a/sql/sql_class.h b/sql/sql_class.h index 5fb6a6c575e..b583c53013e 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -4903,12 +4903,13 @@ class THD :public Statement, } /* Data and methods for buld INSERT IDs reporting */ - DYNAMIC_ARRAY *insert_ids; - ulong collect_auto_increment_increment; + //DYNAMIC_ARRAY *insert_ids; + insert_id_desc *insert_id_info; bool init_collecting_insert_id(); + bool send_collected_insert_id(); bool collect_insert_id(ulonglong id); bool report_collected_insert_id(); - void stop_collecting_insert_id(); + void stop_collecting_insert_id() { insert_id_info= NULL; } }; inline void add_to_active_threads(THD *thd)

1 0

[Commits] 07f5d03: MDEV-5313 Improving merge audit api.
by holyfoot＠askmonty.org 02 Dec '18

02 Dec '18

revision-id: 07f5d036fbb5bdcb011a84f5c882a062d9e609e7 (mariadb-10.4.0-54-g07f5d03) parent(s): 3afae13b548c903d86a55530d59fbf7d8666281f committer: Alexey Botchkov timestamp: 2018-12-03 02:35:52 +0400 message: MDEV-5313 Improving merge audit api. service_json and service_cfg_table interfaces added. --- include/mysql/plugin.h | 2 +- include/mysql/service_cfg_table.h | 109 ++++++++++++++++++++++++++++++++++++++ include/mysql/service_json.h | 109 ++++++++++++++++++++++++++++++++++++++ include/mysql/services.h | 2 + include/service_versions.h | 2 + libservices/CMakeLists.txt | 2 + libservices/cfg_table_service.c | 18 +++++++ libservices/json_service.c | 19 +++++++ sql/sql_plugin_services.ic | 20 ++++++- 9 files changed, 281 insertions(+), 2 deletions(-) diff --git a/include/mysql/plugin.h b/include/mysql/plugin.h index b122d88..02a87c6 100644 --- a/include/mysql/plugin.h +++ b/include/mysql/plugin.h @@ -75,7 +75,7 @@ typedef struct st_mysql_xid MYSQL_XID; #define MYSQL_PLUGIN_INTERFACE_VERSION 0x0104 /* MariaDB plugin interface version */ -#define MARIA_PLUGIN_INTERFACE_VERSION 0x010d +#define MARIA_PLUGIN_INTERFACE_VERSION 0x010e /* The allowable types of plugins diff --git a/include/mysql/service_cfg_table.h b/include/mysql/service_cfg_table.h new file mode 100644 index 0000000..36660ff --- /dev/null +++ b/include/mysql/service_cfg_table.h @@ -0,0 +1,109 @@ +/* Copyright (C) 2018 MariaDB Corporation + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ + +#ifndef MYSQL_SERVICE_CFG_TABLE +#define MYSQL_SERVICE_CFG_TABLE + +/** + @file + cfg table service + + Reading plugin settings from the server table. + + This service lets the plugin to read it's settings + from the server table. + This 'cfg' table has two VARCHAR fields per row + which are + 'id' - the name of the setting + 'value' - the value of the respective 'id' setting + The structure 'cfg_table_row' reflects this format for + the data interchange with the service. + Fuctions of the service: + cfg_table_check_exists + checks if the table exists. + cfg_table_create - creates the table + parameers: + table_name - the name of the table + int id_len, int value_len - the length of the 'id' and 'value' + fields. + const struct cfg_table_row *defaults - table will be filled + with these default rows. + The list of rows is ended by {0,0} row + + cfg_table_set - replaces (inserts if not exists) the setting + parameters: + table_name - the name of the table + const char *id, const char *value - the 'id' and the new value + + cfg_table_get - reads the cft tale content + It return the number of rows in the table or -1 if error. + parameters: + table_name, + struct cfg_table_row *data_buf - buffer to store the records + int n_buf_row - the size of the buffer +*/ + + +#ifdef __cplusplus +extern "C" { +#endif + + +struct cfg_table_row +{ + const char *id; + const char *value; +}; + + +extern struct cfg_table_service_st { + int (*cfg_table_check_exists)(const char *table_name); + int (*cfg_table_create)(const char *table_name, + int id_len, int value_len, + const struct cfg_table_row *defaults); + int (*cfg_table_set)(const char *table_name, + const char *id, const char *value); + int (*cfg_table_get)(const char *table_name, + struct cfg_table_row *data_buf, int n_buf_row); +} *cfg_table_service; + +#ifdef MYSQL_DYNAMIC_PLUGIN + +#define cfg_table_check_exists cfg_table_service->cfg_table_check_exists +#define cfg_table_create cfg_table_service->cfg_table_create +#define cfg_table_set cfg_table_service->cfg_table_set +#define cfg_table_get cfg_table_service->cfg_table_get + +#else + +int cfg_table_check_exists(const char *table_name); +int cfg_table_create(const char *table_name, + int id_len, int value_len, + const struct cfg_table_row *defaults); +int cfg_table_set(const char *table_name, + const char *id, const char *value); +int cfg_table_get(const char *table_name, + struct cfg_table_row *data_buf, int n_buf_row); + +#endif + + +#ifdef __cplusplus +} +#endif + +#endif /*MYSQL_SERVICE_CFG_TABLE */ + + diff --git a/include/mysql/service_json.h b/include/mysql/service_json.h new file mode 100644 index 0000000..5d0e260 --- /dev/null +++ b/include/mysql/service_json.h @@ -0,0 +1,109 @@ +/* Copyright (C) 2018 MariaDB Corporation + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ + +#ifndef MYSQL_SERVICE_JSON +#define MYSQL_SERVICE_JSON + +/** + @file + json service + + Esports JSON parsing methods for plugins to use. + + Fuctions of the service: + js_type - returns the type of the JSON argument, + and the parsed value if it's scalar (not object or array) + + js_get_array_item - expecs JSON array as an argument, + and returns the n_item's item's type and value + Returns JSV_NOTHING type if the array is shorter + than n_item and the actual length of the array in v_len. + + js_get_object_key - expects JSON object as an argument, + searches for a key in the object, return it's type and value. + JSV_NOTHING if no such key found, the number of keys + in v_len. + + js_get_object_nkey - expects JSON object as an argument. + finds n_key's key in the object, returns it's name, type and value. + JSV_NOTHING if object has less keys than n_key. +*/ + + +#ifdef __cplusplus +extern "C" { +#endif + +enum js_value_types +{ + JSV_NOTTHING=0, + JSV_OBJECT=1, + JSV_ARRAY=2, + JSV_STRING=3, + JSV_NUMBER=4, + JSV_TRUE=5, + JSV_FALSE=6, + JSV_NULL=7 +}; + +extern struct json_service_st { + int (*js_type)(const char *js, const char *js_end, + enum js_value_types *vt, + const char **v, int *vlen); + int (*js_get_array_item)(const char *js, const char *js_end, int n_item, + enum js_value_types *vt, + const char **v, int *vlen); + int (*js_get_object_key)(const char *js, const char *js_end, const char *key, + enum js_value_types *vt, + const char **v, int *vlen); + int (*js_get_object_nkey)(const char *js,const char *js_end, int nkey, + const char **keyname, const char **keyname_end, + enum js_value_types *vt, + const char **v, int *vlen); +} *json_service; + +#ifdef MYSQL_DYNAMIC_PLUGIN + +#define js_type json_service->js_type +#define js_get_array_item json_service->js_get_array_item +#define js_get_object_key json_service->js_get_object_key +#define js_get_object_nkey json_service->js_get_object_nkey + +#else + +int js_type(const char *js, const char *js_end, + enum js_value_types *vt, + const char **v, int *vlen); +int js_get_array_item(const char *js, const char *js_end, int n_item, + enum js_value_types *vt, + const char **v, int *vlen); +int js_get_object_key(const char *js, const char *js_end, const char *key, + enum js_value_types *vt, + const char **v, int *vlen); +int js_get_object_nkey(const char *js,const char *js_end, int nkey, + const char **keyname, const char **keyname_end, + enum js_value_types *vt, + const char **v, int *vlen); + +#endif + + +#ifdef __cplusplus +} +#endif + +#endif /*MYSQL_SERVICE_JSON */ + + diff --git a/include/mysql/services.h b/include/mysql/services.h index 6168c5e..2e1879c 100644 --- a/include/mysql/services.h +++ b/include/mysql/services.h @@ -39,6 +39,8 @@ extern "C" { #include <mysql/service_thd_specifics.h> #include <mysql/service_thd_timezone.h> #include <mysql/service_thd_wait.h> +#include <mysql/service_cfg_table.h> +#include <mysql/service_json.h> /*#include <mysql/service_wsrep.h>*/ #ifdef __cplusplus diff --git a/include/service_versions.h b/include/service_versions.h index 753d644..0ea0c3a 100644 --- a/include/service_versions.h +++ b/include/service_versions.h @@ -42,3 +42,5 @@ #define VERSION_thd_timezone 0x0100 #define VERSION_thd_wait 0x0100 #define VERSION_wsrep 0x0202 +#define VERSION_cfg_table 0x0100 +#define VERSION_json 0x0100 diff --git a/libservices/CMakeLists.txt b/libservices/CMakeLists.txt index e20be6d..ac66295 100644 --- a/libservices/CMakeLists.txt +++ b/libservices/CMakeLists.txt @@ -37,6 +37,8 @@ SET(MYSQLSERVICES_SOURCES thd_timezone_service.c thd_wait_service.c wsrep_service.c + cfg_table_service.c + json_service.c ) ADD_CONVENIENCE_LIBRARY(mysqlservices ${MYSQLSERVICES_SOURCES}) diff --git a/libservices/cfg_table_service.c b/libservices/cfg_table_service.c new file mode 100644 index 0000000..8b5bd6f --- /dev/null +++ b/libservices/cfg_table_service.c @@ -0,0 +1,18 @@ +/* Copyright (c) 2018, Monty Program Ab + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include <service_versions.h> +SERVICE_VERSION cfg_table_service= (void*)VERSION_cfg_table; diff --git a/libservices/json_service.c b/libservices/json_service.c new file mode 100644 index 0000000..96b3b3f --- /dev/null +++ b/libservices/json_service.c @@ -0,0 +1,19 @@ + +/* Copyright (c) 2018, Monty Program Ab + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include <service_versions.h> +SERVICE_VERSION json_service= (void*)VERSION_json; diff --git a/sql/sql_plugin_services.ic b/sql/sql_plugin_services.ic index 8de53aa..9aaadb3 100644 --- a/sql/sql_plugin_services.ic +++ b/sql/sql_plugin_services.ic @@ -217,6 +217,22 @@ static struct my_print_error_service_st my_print_error_handler= my_printv_error }; +struct cfg_table_service_st cfg_table_handler= +{ + cfg_table_check_exists, + cfg_table_create, + cfg_table_set, + cfg_table_get +}; + +struct json_service_st json_handler= +{ + js_type, + js_get_array_item, + js_get_object_key, + js_get_object_nkey +}; + static struct st_service_ref list_of_services[]= { { "base64_service", VERSION_base64, &base64_handler }, @@ -239,6 +255,8 @@ static struct st_service_ref list_of_services[]= { "thd_specifics_service", VERSION_thd_specifics, &thd_specifics_handler }, { "thd_timezone_service", VERSION_thd_timezone, &thd_timezone_handler }, { "thd_wait_service", VERSION_thd_wait, &thd_wait_handler }, - { "wsrep_service", VERSION_wsrep, &wsrep_handler } + { "wsrep_service", VERSION_wsrep, &wsrep_handler }, + { "cfg_table_service", VERSION_cfg_table, &cfg_table_handler }, + { "json_service", VERSION_json, &json_handler } };

1 0

[Commits] 9ae6a29c90a: Remove debug printout
by Sergei Petrunia 02 Dec '18

02 Dec '18

revision-id: 9ae6a29c90a1d6d1c7eaa2e642dd3e876957d0a2 (fb-prod201801-173-g9ae6a29c90a) parent(s): 88f9846f17bdcd1e733f784ee844c9c2f6535ab3 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2018-12-02 20:13:25 +0300 message: Remove debug printout --- storage/rocksdb/ha_rocksdb.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index d5d8ead4883..3ec6b189f8b 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -8136,7 +8136,6 @@ void ha_rocksdb::set_range_lock(Rdb_transaction *tx, if (m_lock_rows != RDB_LOCK_WRITE || !rocksdb_use_range_locking) { return; } - fprintf(stderr, "Setting a range lock\n"); if (find_flag == HA_READ_KEY_EXACT) { end_slice= slice;

1 0

[Commits] 88f9846f17b: Update to latest range-locking-RocksDB
by Sergei Petrunia 02 Dec '18

02 Dec '18

revision-id: 88f9846f17bdcd1e733f784ee844c9c2f6535ab3 (fb-prod201801-172-g88f9846f17b) parent(s): 2f7634aa98f2413a383db8fd5a848571a134eb13 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2018-12-02 20:11:27 +0300 message: Update to latest range-locking-RocksDB --- rocksdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocksdb b/rocksdb index 0320154e8fa..574dd4e574d 160000 --- a/rocksdb +++ b/rocksdb @@ -1 +1 @@ -Subproject commit 0320154e8fa4a3a24f1a65fec2487e490dfb8507 +Subproject commit 574dd4e574d60263db9d37faede89057caefafca

1 0

[Commits] 7547e0ec336: Update rocksdb to newest one with Range Locking
by Sergei Petrunia 02 Dec '18

02 Dec '18

revision-id: 7547e0ec3364d02416d5e1a3a497aee9b454ec0e (fb-prod201801-172-g7547e0ec336) parent(s): 2f7634aa98f2413a383db8fd5a848571a134eb13 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2018-12-02 19:57:00 +0300 message: Update rocksdb to newest one with Range Locking --- rocksdb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocksdb b/rocksdb index 0320154e8fa..574dd4e574d 160000 --- a/rocksdb +++ b/rocksdb @@ -1 +1 @@ -Subproject commit 0320154e8fa4a3a24f1a65fec2487e490dfb8507 +Subproject commit 574dd4e574d60263db9d37faede89057caefafca

1 0

[Commits] 82413942bcf: Update rocksdb, now we get proper error code on deadlock
by Sergei Petrunia 02 Dec '18

02 Dec '18

revision-id: 82413942bcf691dea1975e9674558b4c0f9de9bd (fb-prod201801-171-g82413942bcf) parent(s): ed5c828882cf85ffa53e33c350d9cebb10f72a4b author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2018-12-02 19:54:28 +0300 message: Update rocksdb, now we get proper error code on deadlock rocksdb.rqg_transactions 'range_locking' --- rocksdb | 2 +- storage/rocksdb/ha_rocksdb.cc | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/rocksdb b/rocksdb index 0ba14a1efa8..574dd4e574d 160000 --- a/rocksdb +++ b/rocksdb @@ -1 +1 @@ -Subproject commit 0ba14a1efa8f4cfccedc85c7efa317be35c60333 +Subproject commit 574dd4e574d60263db9d37faede89057caefafca diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc index d5d8ead4883..3ec6b189f8b 100644 --- a/storage/rocksdb/ha_rocksdb.cc +++ b/storage/rocksdb/ha_rocksdb.cc @@ -8136,7 +8136,6 @@ void ha_rocksdb::set_range_lock(Rdb_transaction *tx, if (m_lock_rows != RDB_LOCK_WRITE || !rocksdb_use_range_locking) { return; } - fprintf(stderr, "Setting a range lock\n"); if (find_flag == HA_READ_KEY_EXACT) { end_slice= slice;

1 0