- commits - lists.mariadb.org

[Commits] c0e1ceabbf9: MDEV-23573: SIGSEGV in Column_stat::store_stat_fields
by varun 26 Sep '20

26 Sep '20

revision-id: c0e1ceabbf9b8a824daf36a2bbc29301f545daff (mariadb-10.1.43-293-gc0e1ceabbf9) parent(s): 1be8ac390d3665d061141fe50a1c9e63b7b4b294 author: Varun Gupta committer: Varun Gupta timestamp: 2020-09-26 16:36:52 +0530 message: MDEV-23573: SIGSEGV in Column_stat::store_stat_fields For EITS collection min and max fields are allocated for each column that is set in the read_set bitmap of a table. This allocation of min and max fields happens inside alloc_statistics_for_table. For a partitioned table ha_rnd_init is called inside the function collect_statistics_for_table which sets the read_set bitmap for the columns inside the partition expression. This happens only when there is a write lock on the partitioned table. But the allocation happens before this, so min and max fields are not allocated for the columns involved in the partition expression. This resulted in a crash, as the EITS statistics were collected but there was no min and max field to store the value to. The fix would be to call ha_rnd_init inside the function alloc_statistics_for_table that would make sure that min and max fields are allocated for the columns involved in the partition expression. --- mysql-test/r/stat_tables.result | 10 +++++++ mysql-test/r/stat_tables_innodb.result | 10 +++++++ mysql-test/t/stat_tables.test | 10 +++++++ sql/sql_statistics.cc | 49 +++++++++++++++++----------------- 4 files changed, 55 insertions(+), 24 deletions(-) diff --git a/mysql-test/r/stat_tables.result b/mysql-test/r/stat_tables.result index d26221b5f8d..469bd322ee0 100644 --- a/mysql-test/r/stat_tables.result +++ b/mysql-test/r/stat_tables.result @@ -742,3 +742,13 @@ drop table t1; set @@optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; set @save_optimizer_switch=@@optimizer_switch; set use_stat_tables=@save_use_stat_tables; +# +# MDEV-23753: SIGSEGV in Column_stat::store_stat_fields +# +CREATE TABLE t1 (a INT, b INT) PARTITION BY HASH (b) PARTITIONS 2; +LOCK TABLES t1 WRITE; +ANALYZE TABLE t1 PERSISTENT FOR COLUMNS (a) INDEXES (); +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +DROP TABLE t1; diff --git a/mysql-test/r/stat_tables_innodb.result b/mysql-test/r/stat_tables_innodb.result index fd9f337c30d..d3dcffb14c7 100644 --- a/mysql-test/r/stat_tables_innodb.result +++ b/mysql-test/r/stat_tables_innodb.result @@ -769,6 +769,16 @@ drop table t1; set @@optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity; set @save_optimizer_switch=@@optimizer_switch; set use_stat_tables=@save_use_stat_tables; +# +# MDEV-23753: SIGSEGV in Column_stat::store_stat_fields +# +CREATE TABLE t1 (a INT, b INT) PARTITION BY HASH (b) PARTITIONS 2; +LOCK TABLES t1 WRITE; +ANALYZE TABLE t1 PERSISTENT FOR COLUMNS (a) INDEXES (); +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +DROP TABLE t1; set optimizer_switch=@save_optimizer_switch_for_stat_tables_test; # # MDEV-22851: Engine independent index statistics are incorrect for large tables on Windows. diff --git a/mysql-test/t/stat_tables.test b/mysql-test/t/stat_tables.test index e9f37698a73..df16b3c9403 100644 --- a/mysql-test/t/stat_tables.test +++ b/mysql-test/t/stat_tables.test @@ -1,4 +1,5 @@ --source include/have_stat_tables.inc +--source include/have_partition.inc select @@global.use_stat_tables; select @@session.use_stat_tables; @@ -503,3 +504,12 @@ set @@optimizer_use_condition_selectivity= @save_optimizer_use_condition_selecti set @save_optimizer_switch=@@optimizer_switch; set use_stat_tables=@save_use_stat_tables; + +--echo # +--echo # MDEV-23753: SIGSEGV in Column_stat::store_stat_fields +--echo # + +CREATE TABLE t1 (a INT, b INT) PARTITION BY HASH (b) PARTITIONS 2; +LOCK TABLES t1 WRITE; +ANALYZE TABLE t1 PERSISTENT FOR COLUMNS (a) INDEXES (); +DROP TABLE t1; diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc index c2581fd6105..1f1ef41a9dd 100644 --- a/sql/sql_statistics.cc +++ b/sql/sql_statistics.cc @@ -2099,6 +2099,9 @@ int alloc_statistics_for_table(THD* thd, TABLE *table) ulonglong *idx_avg_frequency= (ulonglong*) alloc_root(&table->mem_root, sizeof(ulonglong) * key_parts); + if (table->file->ha_rnd_init(TRUE)) + DBUG_RETURN(1); + uint columns= 0; for (field_ptr= table->field; *field_ptr; field_ptr++) { @@ -2591,36 +2594,34 @@ int collect_statistics_for_table(THD *thd, TABLE *table) restore_record(table, s->default_values); /* Perform a full table scan to collect statistics on 'table's columns */ - if (!(rc= file->ha_rnd_init(TRUE))) - { - DEBUG_SYNC(table->in_use, "statistics_collection_start"); - while ((rc= file->ha_rnd_next(table->record[0])) != HA_ERR_END_OF_FILE) - { - if (thd->killed) - break; + DEBUG_SYNC(table->in_use, "statistics_collection_start"); - if (rc) - { - if (rc == HA_ERR_RECORD_DELETED) - continue; - break; - } + while ((rc= file->ha_rnd_next(table->record[0])) != HA_ERR_END_OF_FILE) + { + if (thd->killed) + break; - for (field_ptr= table->field; *field_ptr; field_ptr++) - { - table_field= *field_ptr; - if (!bitmap_is_set(table->read_set, table_field->field_index)) - continue; - if ((rc= table_field->collected_stats->add(rows))) - break; - } - if (rc) + if (rc) + { + if (rc == HA_ERR_RECORD_DELETED) + continue; + break; + } + + for (field_ptr= table->field; *field_ptr; field_ptr++) + { + table_field= *field_ptr; + if (!bitmap_is_set(table->read_set, table_field->field_index)) + continue; + if ((rc= table_field->collected_stats->add(rows))) break; - rows++; } - file->ha_rnd_end(); + if (rc) + break; + rows++; } + file->ha_rnd_end(); rc= (rc == HA_ERR_END_OF_FILE && !thd->killed) ? 0 : 1; /*

1 0

[Commits] 3080b7eb: MDEV-23811: With large number of indexes optimizer chooses an inefficient plan
by IgorBabaev 25 Sep '20

25 Sep '20

revision-id: 3080b7ebe3cf17107f5b21ac135b0e01fef04fe9 (mariadb-10.2.31-462-g3080b7eb) parent(s): 7c5519c12d46ead947d341cbdcbb6fbbe4d4fe1b author: Igor Babaev committer: Igor Babaev timestamp: 2020-09-24 22:02:00 -0700 message: MDEV-23811: With large number of indexes optimizer chooses an inefficient plan This bug could manifest itself for a query with WHERE condition containing top level OR formula such that each conjunct contained a single-range condition supported by the same index. One of these range conditions must be fully covered by another range condition that is used later in the OR formula. Additionally at least one of these condition should be ANDed with a sargable range condition supported by a different index. There were several attempts to fix related problems for OR conditions after the backport of range optimizer code from MySQL (commit 0e19f3e36f7842583feb6bead2c2600cd620bced). Unfortunately the first of these fixes contained typo remained unnoticed until recently. This typo bug led to rejection of valid range accesses. This patch fixed this typo bug. The fix revealed another two bugs: one in a constructor for SEL_ARG, the other in the function tree_or(). Both are fixed in this patch. --- mysql-test/r/range.result | 74 ++++++++++++++++++++++++- mysql-test/r/range_mrr_icp.result | 74 ++++++++++++++++++++++++- mysql-test/r/range_vs_index_merge_innodb.result | 2 +- mysql-test/t/range.test | 42 ++++++++++++++ sql/opt_range.cc | 20 ++++--- 5 files changed, 201 insertions(+), 11 deletions(-) diff --git a/mysql-test/r/range.result b/mysql-test/r/range.result index 26ea2c6..7299982 100644 --- a/mysql-test/r/range.result +++ b/mysql-test/r/range.result @@ -1274,7 +1274,7 @@ SELECT * FROM t1 WHERE 5 <= a AND b = 3 OR 3 <= a; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range a a 5 NULL 4 Using where; Using index +1 SIMPLE t1 range a a 5 NULL 3 Using where; Using index SELECT * FROM t1 WHERE 3 <= a AND a <= 5 OR 5 <= a AND b = 3 OR @@ -3054,5 +3054,77 @@ a b set eq_range_index_dive_limit=default; drop table t1; # +# MDEV-23811: Both disjunct of WHERE condition contain range conditions +# for the same index such that the second range condition +# fully covers the first one. Additionally one of the disjuncts +# contains a range condition for the other index. +# +create table t1 ( +pk int primary key auto_increment, a int, b int, +index idx1(a), index idx2(b) +); +insert into t1(a,b) values +(5,50), (1,10), (3,30), (7,70), (8,80), (4,40), (2,20), (6,60); +insert into t1(a,b) select a+10, b+100 from t1; +insert into t1(a,b) select a+20, b+200 from t1; +insert into t1(a,b) select a+30, b+300 from t1; +insert into t1(a,b) select a,b from t1; +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status OK +explain select * from t1 where ((a between 3 and 4) and b < 100) or (a between 2 and 5); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx1,idx2 idx1 5 NULL 11 Using index condition; Using where +select * from t1 where ((a between 3 and 4) and b < 100) or (a between 2 and 5); +pk a b +7 2 20 +71 2 20 +3 3 30 +67 3 30 +6 4 40 +70 4 40 +1 5 50 +65 5 50 +explain select * from t1 where (a between 2 and 5) or ((a between 3 and 4) and b < 100); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx1,idx2 idx1 5 NULL 11 Using index condition; Using where +select * from t1 where (a between 2 and 5) or ((a between 3 and 4) and b < 100); +pk a b +7 2 20 +71 2 20 +3 3 30 +67 3 30 +6 4 40 +70 4 40 +1 5 50 +65 5 50 +explain select * from t1 where (a between 3 and 4) or ((a between 2 and 5) and b < 100); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx1,idx2 idx1 5 NULL 11 Using index condition; Using where +select * from t1 where (a between 3 and 4) or ((a between 2 and 5) and b < 100); +pk a b +7 2 20 +71 2 20 +3 3 30 +67 3 30 +6 4 40 +70 4 40 +1 5 50 +65 5 50 +explain select * from t1 where ((a between 2 and 5) and b < 100) or (a between 3 and 4); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx1,idx2 idx1 5 NULL 11 Using index condition; Using where +select * from t1 where ((a between 2 and 5) and b < 100) or (a between 3 and 4); +pk a b +7 2 20 +71 2 20 +3 3 30 +67 3 30 +6 4 40 +70 4 40 +1 5 50 +65 5 50 +drop table t1; +# # End of 10.2 tests # diff --git a/mysql-test/r/range_mrr_icp.result b/mysql-test/r/range_mrr_icp.result index fe4eb99..cdaaa5e 100644 --- a/mysql-test/r/range_mrr_icp.result +++ b/mysql-test/r/range_mrr_icp.result @@ -1276,7 +1276,7 @@ SELECT * FROM t1 WHERE 5 <= a AND b = 3 OR 3 <= a; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range a a 5 NULL 4 Using where; Using index +1 SIMPLE t1 range a a 5 NULL 3 Using where; Using index SELECT * FROM t1 WHERE 3 <= a AND a <= 5 OR 5 <= a AND b = 3 OR @@ -3066,6 +3066,78 @@ a b set eq_range_index_dive_limit=default; drop table t1; # +# MDEV-23811: Both disjunct of WHERE condition contain range conditions +# for the same index such that the second range condition +# fully covers the first one. Additionally one of the disjuncts +# contains a range condition for the other index. +# +create table t1 ( +pk int primary key auto_increment, a int, b int, +index idx1(a), index idx2(b) +); +insert into t1(a,b) values +(5,50), (1,10), (3,30), (7,70), (8,80), (4,40), (2,20), (6,60); +insert into t1(a,b) select a+10, b+100 from t1; +insert into t1(a,b) select a+20, b+200 from t1; +insert into t1(a,b) select a+30, b+300 from t1; +insert into t1(a,b) select a,b from t1; +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status OK +explain select * from t1 where ((a between 3 and 4) and b < 100) or (a between 2 and 5); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx1,idx2 idx1 5 NULL 11 Using index condition; Using where; Rowid-ordered scan +select * from t1 where ((a between 3 and 4) and b < 100) or (a between 2 and 5); +pk a b +1 5 50 +3 3 30 +6 4 40 +7 2 20 +65 5 50 +67 3 30 +70 4 40 +71 2 20 +explain select * from t1 where (a between 2 and 5) or ((a between 3 and 4) and b < 100); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx1,idx2 idx1 5 NULL 11 Using index condition; Using where; Rowid-ordered scan +select * from t1 where (a between 2 and 5) or ((a between 3 and 4) and b < 100); +pk a b +1 5 50 +3 3 30 +6 4 40 +7 2 20 +65 5 50 +67 3 30 +70 4 40 +71 2 20 +explain select * from t1 where (a between 3 and 4) or ((a between 2 and 5) and b < 100); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx1,idx2 idx1 5 NULL 11 Using index condition; Using where; Rowid-ordered scan +select * from t1 where (a between 3 and 4) or ((a between 2 and 5) and b < 100); +pk a b +1 5 50 +3 3 30 +6 4 40 +7 2 20 +65 5 50 +67 3 30 +70 4 40 +71 2 20 +explain select * from t1 where ((a between 2 and 5) and b < 100) or (a between 3 and 4); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx1,idx2 idx1 5 NULL 11 Using index condition; Using where; Rowid-ordered scan +select * from t1 where ((a between 2 and 5) and b < 100) or (a between 3 and 4); +pk a b +1 5 50 +3 3 30 +6 4 40 +7 2 20 +65 5 50 +67 3 30 +70 4 40 +71 2 20 +drop table t1; +# # End of 10.2 tests # set optimizer_switch=@mrr_icp_extra_tmp; diff --git a/mysql-test/r/range_vs_index_merge_innodb.result b/mysql-test/r/range_vs_index_merge_innodb.result index 581f512..916c30b 100644 --- a/mysql-test/r/range_vs_index_merge_innodb.result +++ b/mysql-test/r/range_vs_index_merge_innodb.result @@ -369,7 +369,7 @@ WHERE ((ID < 200) AND (Name LIKE 'Ha%' OR (Country > 'A' AND Country < 'ARG'))) OR ((ID BETWEEN 100 AND 200) AND (Name LIKE 'Pa%' OR (Population > 103000 AND Population < 104000))); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE City index_merge PRIMARY,Population,Country,Name Name,Population,PRIMARY 39,4,4 NULL 307 Using sort_union(Name,Population,PRIMARY); Using where +1 SIMPLE City range PRIMARY,Population,Country,Name PRIMARY 4 NULL 200 Using where SELECT * FROM City USE INDEX () WHERE ((ID < 10) AND (Name LIKE 'H%' OR (Country > 'A' AND Country < 'ARG'))) OR ((ID BETWEEN 100 AND 110) AND diff --git a/mysql-test/t/range.test b/mysql-test/t/range.test index 67d876d..0d2fbe2 100644 --- a/mysql-test/t/range.test +++ b/mysql-test/t/range.test @@ -2096,6 +2096,48 @@ set eq_range_index_dive_limit=default; drop table t1; --echo # +--echo # MDEV-23811: Both disjunct of WHERE condition contain range conditions +--echo # for the same index such that the second range condition +--echo # fully covers the first one. Additionally one of the disjuncts +--echo # contains a range condition for the other index. +--echo # + +create table t1 ( + pk int primary key auto_increment, a int, b int, + index idx1(a), index idx2(b) +); +insert into t1(a,b) values + (5,50), (1,10), (3,30), (7,70), (8,80), (4,40), (2,20), (6,60); +insert into t1(a,b) select a+10, b+100 from t1; +insert into t1(a,b) select a+20, b+200 from t1; +insert into t1(a,b) select a+30, b+300 from t1; +insert into t1(a,b) select a,b from t1; + +analyze table t1; + +let $q1= +select * from t1 where ((a between 3 and 4) and b < 100) or (a between 2 and 5); +eval explain $q1; +eval $q1; + +let $q2= +select * from t1 where (a between 2 and 5) or ((a between 3 and 4) and b < 100); +eval explain $q2; +eval $q2; + +let $q3= +select * from t1 where (a between 3 and 4) or ((a between 2 and 5) and b < 100); +eval explain $q3; +eval $q3; + +let $q4= +select * from t1 where ((a between 2 and 5) and b < 100) or (a between 3 and 4); +eval explain $q4; +eval $q4; + +drop table t1; + +--echo # --echo # End of 10.2 tests --echo # diff --git a/sql/opt_range.cc b/sql/opt_range.cc index e933d2a..cd58202 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -1852,6 +1852,9 @@ SEL_ARG::SEL_ARG(SEL_ARG &arg) :Sql_alloc() next_key_part=arg.next_key_part; max_part_no= arg.max_part_no; use_count=1; elements=1; + next= 0; + if (next_key_part) + ++next_key_part->use_count; } @@ -8869,9 +8872,15 @@ tree_or(RANGE_OPT_PARAM *param,SEL_TREE *tree1,SEL_TREE *tree2) } bool no_imerge_from_ranges= FALSE; + SEL_TREE *rt1= tree1; + SEL_TREE *rt2= tree2; /* Build the range part of the tree for the formula (1) */ if (sel_trees_can_be_ored(param, tree1, tree2, &ored_keys)) { + if (no_merges1) + rt1= new SEL_TREE(tree1, TRUE, param); + if (no_merges2) + rt2= new SEL_TREE(tree2, TRUE, param); bool must_be_ored= sel_trees_must_be_ored(param, tree1, tree2, ored_keys); no_imerge_from_ranges= must_be_ored; @@ -8929,12 +8938,6 @@ tree_or(RANGE_OPT_PARAM *param,SEL_TREE *tree1,SEL_TREE *tree2) else if (!no_ranges1 && !no_ranges2 && !no_imerge_from_ranges) { /* Build the imerge part of the tree for the formula (1) */ - SEL_TREE *rt1= tree1; - SEL_TREE *rt2= tree2; - if (no_merges1) - rt1= new SEL_TREE(tree1, TRUE, param); - if (no_merges2) - rt2= new SEL_TREE(tree2, TRUE, param); if (!rt1 || !rt2 || result->merges.push_back(imerge_from_ranges) || imerge_from_ranges->or_sel_tree(param, rt1) || @@ -9597,10 +9600,11 @@ key_or(RANGE_OPT_PARAM *param, SEL_ARG *key1,SEL_ARG *key2) if (!tmp->next_key_part) { + SEL_ARG *key2_next= key2->next; if (key2->use_count) { SEL_ARG *key2_cpy= new SEL_ARG(*key2); - if (key2_cpy) + if (!key2_cpy) return 0; key2= key2_cpy; } @@ -9621,7 +9625,7 @@ key_or(RANGE_OPT_PARAM *param, SEL_ARG *key1,SEL_ARG *key2) Move on to next range in key2 */ key2->increment_use_count(-1); // Free not used tree - key2=key2->next; + key2=key2_next; continue; } else

1 0

[Commits] b0a9bfc: MDEV-23811: With large number of indexes optimizer chooses an inefficient plan
by IgorBabaev 25 Sep '20

25 Sep '20

revision-id: b0a9bfcf6d187a5de2ebda15d09b3f3779bff264 (mariadb-10.2.31-462-gb0a9bfc) parent(s): 7c5519c12d46ead947d341cbdcbb6fbbe4d4fe1b author: Igor Babaev committer: Igor Babaev timestamp: 2020-09-24 19:56:36 -0700 message: MDEV-23811: With large number of indexes optimizer chooses an inefficient plan This bug could manifest itself for a query with WHERE condition containing top level OR formula such that each conjunct contained a single-range condition supported by the same index. One of these range conditions must be fully covered by another range condition that is used later in the OR formula. Additionally at leat one of these condition should be ANDed with a sargable range condition supported by a different index. There were several attempts to fix related problems for OR conditions after the backport of range optimizer code from MySQL (commit 0e19f3e36f7842583feb6bead2c2600cd620bced). Unfortunately the first of these fixes contained typo remained unnoticed until recently. This typo bug led to rejection of valid range accesses. This patch fixed this typo bug. The fix revealed another two bugs: one in a constructor for SEL_ARG, the other in the function tree_or(). Both are fixed in this patch. --- mysql-test/r/range.result | 74 ++++++++++++++++++++++++- mysql-test/r/range_mrr_icp.result | 74 ++++++++++++++++++++++++- mysql-test/r/range_vs_index_merge_innodb.result | 2 +- mysql-test/t/range.test | 42 ++++++++++++++ sql/opt_range.cc | 20 ++++--- 5 files changed, 201 insertions(+), 11 deletions(-) diff --git a/mysql-test/r/range.result b/mysql-test/r/range.result index 26ea2c6..7299982 100644 --- a/mysql-test/r/range.result +++ b/mysql-test/r/range.result @@ -1274,7 +1274,7 @@ SELECT * FROM t1 WHERE 5 <= a AND b = 3 OR 3 <= a; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range a a 5 NULL 4 Using where; Using index +1 SIMPLE t1 range a a 5 NULL 3 Using where; Using index SELECT * FROM t1 WHERE 3 <= a AND a <= 5 OR 5 <= a AND b = 3 OR @@ -3054,5 +3054,77 @@ a b set eq_range_index_dive_limit=default; drop table t1; # +# MDEV-23811: Both disjunct of WHERE condition contain range conditions +# for the same index such that the second range condition +# fully covers the first one. Additionally one of the disjuncts +# contains a range condition for the other index. +# +create table t1 ( +pk int primary key auto_increment, a int, b int, +index idx1(a), index idx2(b) +); +insert into t1(a,b) values +(5,50), (1,10), (3,30), (7,70), (8,80), (4,40), (2,20), (6,60); +insert into t1(a,b) select a+10, b+100 from t1; +insert into t1(a,b) select a+20, b+200 from t1; +insert into t1(a,b) select a+30, b+300 from t1; +insert into t1(a,b) select a,b from t1; +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status OK +explain select * from t1 where ((a between 3 and 4) and b < 100) or (a between 2 and 5); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx1,idx2 idx1 5 NULL 11 Using index condition; Using where +select * from t1 where ((a between 3 and 4) and b < 100) or (a between 2 and 5); +pk a b +7 2 20 +71 2 20 +3 3 30 +67 3 30 +6 4 40 +70 4 40 +1 5 50 +65 5 50 +explain select * from t1 where (a between 2 and 5) or ((a between 3 and 4) and b < 100); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx1,idx2 idx1 5 NULL 11 Using index condition; Using where +select * from t1 where (a between 2 and 5) or ((a between 3 and 4) and b < 100); +pk a b +7 2 20 +71 2 20 +3 3 30 +67 3 30 +6 4 40 +70 4 40 +1 5 50 +65 5 50 +explain select * from t1 where (a between 3 and 4) or ((a between 2 and 5) and b < 100); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx1,idx2 idx1 5 NULL 11 Using index condition; Using where +select * from t1 where (a between 3 and 4) or ((a between 2 and 5) and b < 100); +pk a b +7 2 20 +71 2 20 +3 3 30 +67 3 30 +6 4 40 +70 4 40 +1 5 50 +65 5 50 +explain select * from t1 where ((a between 2 and 5) and b < 100) or (a between 3 and 4); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx1,idx2 idx1 5 NULL 11 Using index condition; Using where +select * from t1 where ((a between 2 and 5) and b < 100) or (a between 3 and 4); +pk a b +7 2 20 +71 2 20 +3 3 30 +67 3 30 +6 4 40 +70 4 40 +1 5 50 +65 5 50 +drop table t1; +# # End of 10.2 tests # diff --git a/mysql-test/r/range_mrr_icp.result b/mysql-test/r/range_mrr_icp.result index fe4eb99..cdaaa5e 100644 --- a/mysql-test/r/range_mrr_icp.result +++ b/mysql-test/r/range_mrr_icp.result @@ -1276,7 +1276,7 @@ SELECT * FROM t1 WHERE 5 <= a AND b = 3 OR 3 <= a; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range a a 5 NULL 4 Using where; Using index +1 SIMPLE t1 range a a 5 NULL 3 Using where; Using index SELECT * FROM t1 WHERE 3 <= a AND a <= 5 OR 5 <= a AND b = 3 OR @@ -3066,6 +3066,78 @@ a b set eq_range_index_dive_limit=default; drop table t1; # +# MDEV-23811: Both disjunct of WHERE condition contain range conditions +# for the same index such that the second range condition +# fully covers the first one. Additionally one of the disjuncts +# contains a range condition for the other index. +# +create table t1 ( +pk int primary key auto_increment, a int, b int, +index idx1(a), index idx2(b) +); +insert into t1(a,b) values +(5,50), (1,10), (3,30), (7,70), (8,80), (4,40), (2,20), (6,60); +insert into t1(a,b) select a+10, b+100 from t1; +insert into t1(a,b) select a+20, b+200 from t1; +insert into t1(a,b) select a+30, b+300 from t1; +insert into t1(a,b) select a,b from t1; +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status OK +explain select * from t1 where ((a between 3 and 4) and b < 100) or (a between 2 and 5); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx1,idx2 idx1 5 NULL 11 Using index condition; Using where; Rowid-ordered scan +select * from t1 where ((a between 3 and 4) and b < 100) or (a between 2 and 5); +pk a b +1 5 50 +3 3 30 +6 4 40 +7 2 20 +65 5 50 +67 3 30 +70 4 40 +71 2 20 +explain select * from t1 where (a between 2 and 5) or ((a between 3 and 4) and b < 100); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx1,idx2 idx1 5 NULL 11 Using index condition; Using where; Rowid-ordered scan +select * from t1 where (a between 2 and 5) or ((a between 3 and 4) and b < 100); +pk a b +1 5 50 +3 3 30 +6 4 40 +7 2 20 +65 5 50 +67 3 30 +70 4 40 +71 2 20 +explain select * from t1 where (a between 3 and 4) or ((a between 2 and 5) and b < 100); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx1,idx2 idx1 5 NULL 11 Using index condition; Using where; Rowid-ordered scan +select * from t1 where (a between 3 and 4) or ((a between 2 and 5) and b < 100); +pk a b +1 5 50 +3 3 30 +6 4 40 +7 2 20 +65 5 50 +67 3 30 +70 4 40 +71 2 20 +explain select * from t1 where ((a between 2 and 5) and b < 100) or (a between 3 and 4); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 range idx1,idx2 idx1 5 NULL 11 Using index condition; Using where; Rowid-ordered scan +select * from t1 where ((a between 2 and 5) and b < 100) or (a between 3 and 4); +pk a b +1 5 50 +3 3 30 +6 4 40 +7 2 20 +65 5 50 +67 3 30 +70 4 40 +71 2 20 +drop table t1; +# # End of 10.2 tests # set optimizer_switch=@mrr_icp_extra_tmp; diff --git a/mysql-test/r/range_vs_index_merge_innodb.result b/mysql-test/r/range_vs_index_merge_innodb.result index 581f512..916c30b 100644 --- a/mysql-test/r/range_vs_index_merge_innodb.result +++ b/mysql-test/r/range_vs_index_merge_innodb.result @@ -369,7 +369,7 @@ WHERE ((ID < 200) AND (Name LIKE 'Ha%' OR (Country > 'A' AND Country < 'ARG'))) OR ((ID BETWEEN 100 AND 200) AND (Name LIKE 'Pa%' OR (Population > 103000 AND Population < 104000))); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE City index_merge PRIMARY,Population,Country,Name Name,Population,PRIMARY 39,4,4 NULL 307 Using sort_union(Name,Population,PRIMARY); Using where +1 SIMPLE City range PRIMARY,Population,Country,Name PRIMARY 4 NULL 200 Using where SELECT * FROM City USE INDEX () WHERE ((ID < 10) AND (Name LIKE 'H%' OR (Country > 'A' AND Country < 'ARG'))) OR ((ID BETWEEN 100 AND 110) AND diff --git a/mysql-test/t/range.test b/mysql-test/t/range.test index 67d876d..0d2fbe2 100644 --- a/mysql-test/t/range.test +++ b/mysql-test/t/range.test @@ -2096,6 +2096,48 @@ set eq_range_index_dive_limit=default; drop table t1; --echo # +--echo # MDEV-23811: Both disjunct of WHERE condition contain range conditions +--echo # for the same index such that the second range condition +--echo # fully covers the first one. Additionally one of the disjuncts +--echo # contains a range condition for the other index. +--echo # + +create table t1 ( + pk int primary key auto_increment, a int, b int, + index idx1(a), index idx2(b) +); +insert into t1(a,b) values + (5,50), (1,10), (3,30), (7,70), (8,80), (4,40), (2,20), (6,60); +insert into t1(a,b) select a+10, b+100 from t1; +insert into t1(a,b) select a+20, b+200 from t1; +insert into t1(a,b) select a+30, b+300 from t1; +insert into t1(a,b) select a,b from t1; + +analyze table t1; + +let $q1= +select * from t1 where ((a between 3 and 4) and b < 100) or (a between 2 and 5); +eval explain $q1; +eval $q1; + +let $q2= +select * from t1 where (a between 2 and 5) or ((a between 3 and 4) and b < 100); +eval explain $q2; +eval $q2; + +let $q3= +select * from t1 where (a between 3 and 4) or ((a between 2 and 5) and b < 100); +eval explain $q3; +eval $q3; + +let $q4= +select * from t1 where ((a between 2 and 5) and b < 100) or (a between 3 and 4); +eval explain $q4; +eval $q4; + +drop table t1; + +--echo # --echo # End of 10.2 tests --echo # diff --git a/sql/opt_range.cc b/sql/opt_range.cc index e933d2a..cd58202 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -1852,6 +1852,9 @@ SEL_ARG::SEL_ARG(SEL_ARG &arg) :Sql_alloc() next_key_part=arg.next_key_part; max_part_no= arg.max_part_no; use_count=1; elements=1; + next= 0; + if (next_key_part) + ++next_key_part->use_count; } @@ -8869,9 +8872,15 @@ tree_or(RANGE_OPT_PARAM *param,SEL_TREE *tree1,SEL_TREE *tree2) } bool no_imerge_from_ranges= FALSE; + SEL_TREE *rt1= tree1; + SEL_TREE *rt2= tree2; /* Build the range part of the tree for the formula (1) */ if (sel_trees_can_be_ored(param, tree1, tree2, &ored_keys)) { + if (no_merges1) + rt1= new SEL_TREE(tree1, TRUE, param); + if (no_merges2) + rt2= new SEL_TREE(tree2, TRUE, param); bool must_be_ored= sel_trees_must_be_ored(param, tree1, tree2, ored_keys); no_imerge_from_ranges= must_be_ored; @@ -8929,12 +8938,6 @@ tree_or(RANGE_OPT_PARAM *param,SEL_TREE *tree1,SEL_TREE *tree2) else if (!no_ranges1 && !no_ranges2 && !no_imerge_from_ranges) { /* Build the imerge part of the tree for the formula (1) */ - SEL_TREE *rt1= tree1; - SEL_TREE *rt2= tree2; - if (no_merges1) - rt1= new SEL_TREE(tree1, TRUE, param); - if (no_merges2) - rt2= new SEL_TREE(tree2, TRUE, param); if (!rt1 || !rt2 || result->merges.push_back(imerge_from_ranges) || imerge_from_ranges->or_sel_tree(param, rt1) || @@ -9597,10 +9600,11 @@ key_or(RANGE_OPT_PARAM *param, SEL_ARG *key1,SEL_ARG *key2) if (!tmp->next_key_part) { + SEL_ARG *key2_next= key2->next; if (key2->use_count) { SEL_ARG *key2_cpy= new SEL_ARG(*key2); - if (key2_cpy) + if (!key2_cpy) return 0; key2= key2_cpy; } @@ -9621,7 +9625,7 @@ key_or(RANGE_OPT_PARAM *param, SEL_ARG *key1,SEL_ARG *key2) Move on to next range in key2 */ key2->increment_use_count(-1); // Free not used tree - key2=key2->next; + key2=key2_next; continue; } else

1 0

[Commits] 5d9386892b5: MDEV-4633: multi_source.simple test fails sporadically
by sujatha 08 Sep '20

08 Sep '20

revision-id: 5d9386892b58477c4e28de3e40d930b44f5b42d4 (mariadb-10.1.43-274-g5d9386892b5) parent(s): 3ee2422624ffb3d7ffefff8db7ef9398816299bc author: Sujatha committer: Sujatha timestamp: 2020-09-08 13:10:36 +0530 message: MDEV-4633: multi_source.simple test fails sporadically Analysis: ======== Writes to 'rli->log_space_total' needs to be synchronized, otherwise both SQL_THREAD and IO_THREAD can try to modify the variable simultaneously resulting in incorrect rli->log_space_total. In the current test scenario SQL_THREAD is trying to decrement 'rli->log_space_total' in 'purge_first_log' and IO_THREAD is trying to increment the 'rli->log_space_total' in 'queue_event' simultaneously. Hence test occasionally fails with result mismatch. Fix: === Convert 'rli->log_space_total' variable to atomic type. --- sql/log.cc | 4 +++- sql/log.h | 4 +++- sql/rpl_rli.cc | 14 ++++++++------ sql/slave.cc | 19 +++++++++++++++---- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/sql/log.cc b/sql/log.cc index 8049b94bab1..90d5c46a15f 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -4400,7 +4400,9 @@ int MYSQL_BIN_LOG::purge_first_log(Relay_log_info* rli, bool included) 0, 0, &log_space_reclaimed); mysql_mutex_lock(&rli->log_space_lock); - rli->log_space_total-= log_space_reclaimed; + my_atomic_add64_explicit((volatile int64*)(&rli->log_space_total), + (-(ulonglong)log_space_reclaimed), + MY_MEMORY_ORDER_RELAXED); mysql_cond_broadcast(&rli->log_space_cond); mysql_mutex_unlock(&rli->log_space_lock); diff --git a/sql/log.h b/sql/log.h index 277e5c6f69c..9e3807e9939 100644 --- a/sql/log.h +++ b/sql/log.h @@ -712,7 +712,9 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG char buf1[22],buf2[22]; #endif DBUG_ENTER("harvest_bytes_written"); - (*counter)+=bytes_written; + + my_atomic_add64_explicit((volatile int64*)(counter), bytes_written, + MY_MEMORY_ORDER_RELAXED); DBUG_PRINT("info",("counter: %s bytes_written: %s", llstr(*counter,buf1), llstr(bytes_written,buf2))); bytes_written=0; diff --git a/sql/rpl_rli.cc b/sql/rpl_rli.cc index c196a65809a..8815af4ea4a 100644 --- a/sql/rpl_rli.cc +++ b/sql/rpl_rli.cc @@ -140,8 +140,8 @@ int init_relay_log_info(Relay_log_info* rli, rli->slave_skip_counter=0; rli->abort_pos_wait=0; rli->log_space_limit= relay_log_space_limit; - rli->log_space_total= 0; - + my_atomic_store64_explicit((volatile int64*)(&rli->log_space_total), 0, + MY_MEMORY_ORDER_RELAXED); char pattern[FN_REFLEN]; (void) my_realpath(pattern, slave_load_tmpdir, 0); if (fn_format(pattern, PREFIX_SQL_LOAD, pattern, "", @@ -467,7 +467,8 @@ static inline int add_relay_log(Relay_log_info* rli,LOG_INFO* linfo) linfo->log_file_name); DBUG_RETURN(1); } - rli->log_space_total += s.st_size; + my_atomic_add64_explicit((volatile int64*)(&rli->log_space_total), + s.st_size, MY_MEMORY_ORDER_RELAXED); DBUG_PRINT("info",("log_space_total: %llu", rli->log_space_total)); DBUG_RETURN(0); } @@ -477,7 +478,8 @@ static int count_relay_log_space(Relay_log_info* rli) { LOG_INFO linfo; DBUG_ENTER("count_relay_log_space"); - rli->log_space_total= 0; + my_atomic_store64_explicit((volatile int64*)(&rli->log_space_total), 0, + MY_MEMORY_ORDER_RELAXED); if (rli->relay_log.find_log_pos(&linfo, NullS, 1)) { sql_print_error("Could not find first log while counting relay log space"); @@ -1202,8 +1204,8 @@ int purge_relay_logs(Relay_log_info* rli, THD *thd, bool just_reset, strmake_buf(rli->group_relay_log_name, rli->relay_log.get_log_fname()); strmake_buf(rli->event_relay_log_name, rli->relay_log.get_log_fname()); rli->group_relay_log_pos= rli->event_relay_log_pos= BIN_LOG_HEADER_SIZE; - rli->log_space_total= 0; - + my_atomic_store64_explicit((volatile int64*)(&rli->log_space_total), 0, + MY_MEMORY_ORDER_RELAXED); if (count_relay_log_space(rli)) { *errmsg= "Error counting relay log space"; diff --git a/sql/slave.cc b/sql/slave.cc index 06f2b0d955a..9f4b0f53fb4 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -2371,7 +2371,10 @@ static bool wait_for_relay_log_space(Relay_log_info* rli) &rli->log_space_lock, &stage_waiting_for_relay_log_space, &old_stage); - while (rli->log_space_limit < rli->log_space_total && + while (rli->log_space_limit < + (ulonglong)my_atomic_load64_explicit((volatile int64*) + (&rli->log_space_total), + MY_MEMORY_ORDER_RELAXED) && !(slave_killed=io_slave_killed(mi)) && !rli->ignore_log_space_limit) mysql_cond_wait(&rli->log_space_cond, &rli->log_space_lock); @@ -2912,7 +2915,10 @@ static bool send_show_master_info_data(THD *thd, Master_info *mi, bool full, protocol->store(mi->rli.last_error().message, &my_charset_bin); protocol->store((uint32) mi->rli.slave_skip_counter); protocol->store((ulonglong) mi->rli.group_master_log_pos); - protocol->store((ulonglong) mi->rli.log_space_total); + protocol->store((ulonglong) + my_atomic_load64_explicit((volatile int64*) + (&mi->rli.log_space_total), + MY_MEMORY_ORDER_RELAXED)); protocol->store( mi->rli.until_condition==Relay_log_info::UNTIL_NONE ? "None": @@ -4428,7 +4434,9 @@ Stopping slave I/O thread due to out-of-memory error from master"); #endif if (rli->log_space_limit && rli->log_space_limit < - rli->log_space_total && + (ulonglong) my_atomic_load64_explicit((volatile int64*) + (&rli->log_space_total), + MY_MEMORY_ORDER_RELAXED) && !rli->ignore_log_space_limit) if (wait_for_relay_log_space(rli)) { @@ -7031,7 +7039,10 @@ static Log_event* next_event(rpl_group_info *rgi, ulonglong *event_size) is are able to rotate and purge sometime soon. */ if (rli->log_space_limit && - rli->log_space_limit < rli->log_space_total) + rli->log_space_limit < + (ulonglong) my_atomic_load64_explicit((volatile int64*) + (&rli->log_space_total), + MY_MEMORY_ORDER_RELAXED)) { /* force rotation if not in an unfinished group */ rli->sql_force_rotate_relay= !rli->is_in_group();

1 0

[Commits] 683cb88de49: Update libmarias3 to PR#92: updated PR#91, adds capability to support port number
by Sergei Petrunia 07 Sep '20

07 Sep '20

revision-id: 683cb88de49b53073fa5cd36150b233df3e04352 (mariadb-10.5.4-171-g683cb88de49) parent(s): 5d12b119d7506b6b3e6f85a3d4b7dacbb98df2b5 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-09-07 16:07:11 +0300 message: Update libmarias3 to PR#92: updated PR#91, adds capability to support port number --- storage/maria/libmarias3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/maria/libmarias3 b/storage/maria/libmarias3 index 919024c98b9..7aa450e3a02 160000 --- a/storage/maria/libmarias3 +++ b/storage/maria/libmarias3 @@ -1 +1 @@ -Subproject commit 919024c98b965c978e872da7df0c31db7515035b +Subproject commit 7aa450e3a02a20ef62616666a2b582f4d07fb5b3

1 0

[Commits] 5d12b119d75: Update libmarias3 to PR#91 which adds capability to specify port number
by Sergei Petrunia 05 Sep '20

05 Sep '20

revision-id: 5d12b119d7506b6b3e6f85a3d4b7dacbb98df2b5 (mariadb-10.5.4-170-g5d12b119d75) parent(s): 5020dc0bdc38c7706995f20928160065ff8b40b2 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-09-05 22:24:32 +0300 message: Update libmarias3 to PR#91 which adds capability to specify port number --- storage/maria/libmarias3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/maria/libmarias3 b/storage/maria/libmarias3 index d172e86c162..919024c98b9 160000 --- a/storage/maria/libmarias3 +++ b/storage/maria/libmarias3 @@ -1 +1 @@ -Subproject commit d172e86c16224b4e0229ca6f102e662a2315aeff +Subproject commit 919024c98b965c978e872da7df0c31db7515035b

1 0

[Commits] 01f8db9: Followup to PR#91: use curl_easy_setopt() instead of putting port# into the URI.
by Sergei Petrunia 05 Sep '20

05 Sep '20

revision-id: 01f8db9ccde4318ca924c44caebc941a290b1713 (libmarias3-3.1.2-5-g01f8db9) parent(s): 919024c98b965c978e872da7df0c31db7515035b author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-09-05 20:12:28 +0300 message: Followup to PR#91: use curl_easy_setopt() instead of putting port# into the URI. --- src/request.c | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/src/request.c b/src/request.c index cd819c2..80a32d0 100644 --- a/src/request.c +++ b/src/request.c @@ -51,7 +51,7 @@ static void set_error_nocopy(ms3_st *ms3, char *error) ms3->last_error = error; } -static uint8_t build_request_uri(CURL *curl, const char *base_domain, int port, +static uint8_t build_request_uri(CURL *curl, const char *base_domain, const char *bucket, const char *object, const char *query, bool use_http, uint8_t protocol_version) { @@ -61,14 +61,6 @@ static uint8_t build_request_uri(CURL *curl, const char *base_domain, int port, const char *http_protocol = "http"; const char *https_protocol = "https"; const char *protocol; - char port_str[32]; // ":portnumber" or empty string - - if (port) - { - snprintf(port_str, sizeof(port_str) - 1, ":%d", port); - } - else - port_str[0]= 0; // empty string if (base_domain) { @@ -91,43 +83,43 @@ static uint8_t build_request_uri(CURL *curl, const char *base_domain, int port, if (query) { if (path_parts + strlen(domain) + strlen(bucket) + strlen(object) + strlen( - query) + strlen(port_str) >= MAX_URI_LENGTH - 1) + query) >= MAX_URI_LENGTH - 1) { return MS3_ERR_URI_TOO_LONG; } if (protocol_version == 1) { - snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s%s/%s%s?%s", protocol, - domain, port_str, bucket, + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s/%s%s?%s", protocol, + domain, bucket, object, query); } else { - snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s.%s%s%s?%s", protocol, - bucket, domain, port_str, + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s.%s%s?%s", protocol, + bucket, domain, object, query); } } else { if (path_parts + strlen(domain) + strlen(bucket) + strlen( - object) + strlen(port_str) >= MAX_URI_LENGTH - 1) + object) >= MAX_URI_LENGTH - 1) { return MS3_ERR_URI_TOO_LONG; } if (protocol_version == 1) { - snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s%s/%s%s", protocol, - domain, port_str, + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s/%s%s", protocol, + domain, bucket, object); } else { - snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s.%s%s%s", protocol, - bucket, domain, port_str, + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s.%s%s", protocol, + bucket, domain, object); } } @@ -725,8 +717,8 @@ uint8_t execute_request(ms3_st *ms3, command_t cmd, const char *bucket, ms3->query_buffer); } - res = build_request_uri(curl, ms3->base_domain, ms3->port, bucket, path, - query, ms3->use_http, ms3->protocol_version); + res = build_request_uri(curl, ms3->base_domain, bucket, path, query, + ms3->use_http, ms3->protocol_version); if (res) { @@ -783,6 +775,9 @@ uint8_t execute_request(ms3_st *ms3, command_t cmd, const char *bucket, curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); } + if (ms3->port) + curl_easy_setopt(curl, CURLOPT_PORT, (long)ms3->port); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, body_callback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&mem);

1 0

[Commits] 29a6121: Add an option to specify port number
by Sergei Petrunia 05 Sep '20

05 Sep '20

revision-id: 29a6121396da9fc73e3deee7695f7cb95f7121fe (libmarias3-3.1.2-3-g29a6121) parent(s): d172e86c16224b4e0229ca6f102e662a2315aeff author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-09-05 20:05:44 +0300 message: Add an option to specify port number --- libmarias3/marias3.h | 3 ++- src/marias3.c | 14 ++++++++++++++ src/request.c | 3 +++ src/structs.h | 1 + 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/libmarias3/marias3.h b/libmarias3/marias3.h index abe255b..a6f2bf3 100644 --- a/libmarias3/marias3.h +++ b/libmarias3/marias3.h @@ -80,7 +80,8 @@ enum ms3_set_option_t MS3_OPT_DISABLE_SSL_VERIFY, MS3_OPT_BUFFER_CHUNK_SIZE, MS3_OPT_FORCE_LIST_VERSION, - MS3_OPT_FORCE_PROTOCOL_VERSION + MS3_OPT_FORCE_PROTOCOL_VERSION, + MS3_OPT_PORT_NUMBER }; typedef enum ms3_set_option_t ms3_set_option_t; diff --git a/src/marias3.c b/src/marias3.c index 581b181..cebcbb0 100644 --- a/src/marias3.c +++ b/src/marias3.c @@ -166,6 +166,7 @@ ms3_st *ms3_init(const char *s3key, const char *s3secret, ms3->s3key = ms3_cstrdup(s3key); ms3->s3secret = ms3_cstrdup(s3secret); ms3->region = ms3_cstrdup(region); + ms3->port = 0; /* The default value */ if (base_domain && strlen(base_domain)) { @@ -534,6 +535,19 @@ uint8_t ms3_set_option(ms3_st *ms3, ms3_set_option_t option, void *value) break; } + case MS3_OPT_PORT_NUMBER: + { + int port_number; + + if (!value) + { + return MS3_ERR_PARAMETER; + } + memcpy(&port_number, (void*)value, sizeof(int)); + + ms3->port = port_number; + break; + } default: return MS3_ERR_PARAMETER; } diff --git a/src/request.c b/src/request.c index 425367b..80a32d0 100644 --- a/src/request.c +++ b/src/request.c @@ -775,6 +775,9 @@ uint8_t execute_request(ms3_st *ms3, command_t cmd, const char *bucket, curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); } + if (ms3->port) + curl_easy_setopt(curl, CURLOPT_PORT, (long)ms3->port); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_callback); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, body_callback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&mem); diff --git a/src/structs.h b/src/structs.h index 8d30a57..de80b53 100644 --- a/src/structs.h +++ b/src/structs.h @@ -42,6 +42,7 @@ struct ms3_st char *s3secret; char *region; char *base_domain; + int port; // 0 means "Use default" size_t buffer_chunk_size; CURL *curl; char *last_error;

1 0

[Commits] cd0e4f28fd4: MDEV-23662: Make S3 Storage Engine usable with MinIO
by Sergei Petrunia 03 Sep '20

03 Sep '20

revision-id: cd0e4f28fd45433e98f28933bc3ed103b34b64ac (mariadb-10.5.4-167-gcd0e4f28fd4) parent(s): 0af6e525212f81c2da8185817c6ca2e7c3ac036a author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-09-04 00:15:22 +0300 message: MDEV-23662: Make S3 Storage Engine usable with MinIO Add @@s3_port and @@s3_use_http. One can set these to access the local MinIO, for example. --- mysql-test/suite/s3/basic.result | 2 ++ mysql-test/suite/s3/my.cnf | 12 ++++++++++++ mysql-test/suite/s3/slave.cnf | 12 ++++++++++++ storage/maria/ha_s3.cc | 15 +++++++++++++++ storage/maria/s3_func.c | 6 ++++++ storage/maria/s3_func.h | 2 ++ 6 files changed, 49 insertions(+) diff --git a/mysql-test/suite/s3/basic.result b/mysql-test/suite/s3/basic.result index 9cddca79fc5..cf7d7377d92 100644 --- a/mysql-test/suite/s3/basic.result +++ b/mysql-test/suite/s3/basic.result @@ -94,11 +94,13 @@ s3_pagecache_age_threshold X s3_pagecache_buffer_size X s3_pagecache_division_limit X s3_pagecache_file_hash_size X +s3_port X s3_protocol_version X s3_region X s3_replicate_alter_as_create_select X s3_secret_key X s3_slave_ignore_updates X +s3_use_http X show variables like "s3_slave%"; Variable_name Value s3_slave_ignore_updates OFF diff --git a/mysql-test/suite/s3/my.cnf b/mysql-test/suite/s3/my.cnf index 6f8c27d4d0d..e810449d5b3 100644 --- a/mysql-test/suite/s3/my.cnf +++ b/mysql-test/suite/s3/my.cnf @@ -10,3 +10,15 @@ s3=ON #s3-access-key=... #s3-secret-key=... #s3-region=eu-north-1 + +## +## Configuration for local MinIO +## +s3-host-name="127.0.0.1" +# Note: s3-host-name="localhost" doesn't work. It causes +# libmarias3 to use the wrong variant of the protocol. +s3-bucket=storage-engine +s3-access-key=minioadmin +s3-secret-key=minioadmin +s3-port=9000 +s3-use-http=ON diff --git a/mysql-test/suite/s3/slave.cnf b/mysql-test/suite/s3/slave.cnf index 250a46e6322..c5e7f466226 100644 --- a/mysql-test/suite/s3/slave.cnf +++ b/mysql-test/suite/s3/slave.cnf @@ -6,3 +6,15 @@ s3=ON #s3-access-key=... #s3-secret-key=... #s3-region=eu-north-1 + +## +## Configuration for local MinIO +## +s3-host-name="127.0.0.1" +# Note: s3-host-name="localhost" doesn't work. It causes +# libmarias3 to use the wrong variant of the protocol. +s3-bucket=storage-engine +s3-access-key=minioadmin +s3-secret-key=minioadmin +s3-port=9000 +s3-use-http=ON diff --git a/storage/maria/ha_s3.cc b/storage/maria/ha_s3.cc index 1aae5a5b81f..190d99035a5 100644 --- a/storage/maria/ha_s3.cc +++ b/storage/maria/ha_s3.cc @@ -80,6 +80,8 @@ static ulong s3_pagecache_file_hash_size; static ulonglong s3_pagecache_buffer_size; static char *s3_bucket, *s3_access_key=0, *s3_secret_key=0, *s3_region; static char *s3_host_name; +static int s3_port; +static my_bool s3_use_http; static char *s3_tmp_access_key=0, *s3_tmp_secret_key=0; static my_bool s3_debug= 0, s3_slave_ignore_updates= 0; static my_bool s3_replicate_alter_as_create_select= 0; @@ -181,6 +183,15 @@ static MYSQL_SYSVAR_STR(host_name, s3_host_name, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "AWS host name", 0, 0, DEFAULT_AWS_HOST_NAME); +static MYSQL_SYSVAR_INT(port, s3_port, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "Port number to connect to (0 means use default)", + NULL /*check*/, NULL /*update*/, 0 /*default*/, + 0 /*min*/, 65535 /*max*/, 1 /*blk*/); +static MYSQL_SYSVAR_BOOL(use_http, s3_use_http, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "If true, force use of HTTP protocol", + NULL /*check*/, NULL /*update*/, 0 /*default*/); static MYSQL_SYSVAR_STR(access_key, s3_tmp_access_key, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY | PLUGIN_VAR_MEMALLOC, "AWS access key", @@ -276,6 +287,8 @@ static my_bool s3_info_init(S3_INFO *info) return 1; info->protocol_version= (uint8_t) s3_protocol_version; lex_string_set(&info->host_name, s3_host_name); + info->port= s3_port; + info->use_http= s3_use_http; lex_string_set(&info->access_key, s3_access_key); lex_string_set(&info->secret_key, s3_secret_key); lex_string_set(&info->region, s3_region); @@ -1050,6 +1063,8 @@ static struct st_mysql_sys_var* system_variables[]= { MYSQL_SYSVAR(pagecache_division_limit), MYSQL_SYSVAR(pagecache_file_hash_size), MYSQL_SYSVAR(host_name), + MYSQL_SYSVAR(port), + MYSQL_SYSVAR(use_http), MYSQL_SYSVAR(bucket), MYSQL_SYSVAR(access_key), MYSQL_SYSVAR(secret_key), diff --git a/storage/maria/s3_func.c b/storage/maria/s3_func.c index bbaf048d534..9f40790a371 100644 --- a/storage/maria/s3_func.c +++ b/storage/maria/s3_func.c @@ -157,6 +157,12 @@ ms3_st *s3_open_connection(S3_INFO *s3) if (s3->protocol_version) ms3_set_option(s3_client, MS3_OPT_FORCE_PROTOCOL_VERSION, &s3->protocol_version); + if (s3->port) + ms3_set_option(s3_client, MS3_OPT_PORT_NUMBER, &s3->port); + + if (s3->use_http) + ms3_set_option(s3_client, MS3_OPT_USE_HTTP, NULL); + return s3_client; } diff --git a/storage/maria/s3_func.h b/storage/maria/s3_func.h index 8981517dd4d..bd5275494bc 100644 --- a/storage/maria/s3_func.h +++ b/storage/maria/s3_func.h @@ -45,6 +45,8 @@ typedef struct s3_info { /* Connection strings */ LEX_CSTRING access_key, secret_key, region, bucket, host_name; + int port; // 0 means 'Use default' + my_bool use_http; /* Will be set by caller or by ma_open() */ LEX_CSTRING database, table;

1 0

[Commits] 579b785: Add an option to specify port number
by Sergei Petrunia 03 Sep '20

03 Sep '20

revision-id: 579b785002a6a31ecf84d0391de7d839aa612af0 (libmarias3-3.1.2-3-g579b785) parent(s): d172e86c16224b4e0229ca6f102e662a2315aeff author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2020-09-04 00:01:08 +0300 message: Add an option to specify port number --- libmarias3/marias3.h | 3 ++- src/marias3.c | 14 ++++++++++++++ src/request.c | 34 +++++++++++++++++++++------------- src/structs.h | 1 + 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/libmarias3/marias3.h b/libmarias3/marias3.h index abe255b..a6f2bf3 100644 --- a/libmarias3/marias3.h +++ b/libmarias3/marias3.h @@ -80,7 +80,8 @@ enum ms3_set_option_t MS3_OPT_DISABLE_SSL_VERIFY, MS3_OPT_BUFFER_CHUNK_SIZE, MS3_OPT_FORCE_LIST_VERSION, - MS3_OPT_FORCE_PROTOCOL_VERSION + MS3_OPT_FORCE_PROTOCOL_VERSION, + MS3_OPT_PORT_NUMBER }; typedef enum ms3_set_option_t ms3_set_option_t; diff --git a/src/marias3.c b/src/marias3.c index 581b181..cebcbb0 100644 --- a/src/marias3.c +++ b/src/marias3.c @@ -166,6 +166,7 @@ ms3_st *ms3_init(const char *s3key, const char *s3secret, ms3->s3key = ms3_cstrdup(s3key); ms3->s3secret = ms3_cstrdup(s3secret); ms3->region = ms3_cstrdup(region); + ms3->port = 0; /* The default value */ if (base_domain && strlen(base_domain)) { @@ -534,6 +535,19 @@ uint8_t ms3_set_option(ms3_st *ms3, ms3_set_option_t option, void *value) break; } + case MS3_OPT_PORT_NUMBER: + { + int port_number; + + if (!value) + { + return MS3_ERR_PARAMETER; + } + memcpy(&port_number, (void*)value, sizeof(int)); + + ms3->port = port_number; + break; + } default: return MS3_ERR_PARAMETER; } diff --git a/src/request.c b/src/request.c index 425367b..cd819c2 100644 --- a/src/request.c +++ b/src/request.c @@ -51,7 +51,7 @@ static void set_error_nocopy(ms3_st *ms3, char *error) ms3->last_error = error; } -static uint8_t build_request_uri(CURL *curl, const char *base_domain, +static uint8_t build_request_uri(CURL *curl, const char *base_domain, int port, const char *bucket, const char *object, const char *query, bool use_http, uint8_t protocol_version) { @@ -61,6 +61,14 @@ static uint8_t build_request_uri(CURL *curl, const char *base_domain, const char *http_protocol = "http"; const char *https_protocol = "https"; const char *protocol; + char port_str[32]; // ":portnumber" or empty string + + if (port) + { + snprintf(port_str, sizeof(port_str) - 1, ":%d", port); + } + else + port_str[0]= 0; // empty string if (base_domain) { @@ -83,43 +91,43 @@ static uint8_t build_request_uri(CURL *curl, const char *base_domain, if (query) { if (path_parts + strlen(domain) + strlen(bucket) + strlen(object) + strlen( - query) >= MAX_URI_LENGTH - 1) + query) + strlen(port_str) >= MAX_URI_LENGTH - 1) { return MS3_ERR_URI_TOO_LONG; } if (protocol_version == 1) { - snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s/%s%s?%s", protocol, - domain, bucket, + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s%s/%s%s?%s", protocol, + domain, port_str, bucket, object, query); } else { - snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s.%s%s?%s", protocol, - bucket, domain, + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s.%s%s%s?%s", protocol, + bucket, domain, port_str, object, query); } } else { if (path_parts + strlen(domain) + strlen(bucket) + strlen( - object) >= MAX_URI_LENGTH - 1) + object) + strlen(port_str) >= MAX_URI_LENGTH - 1) { return MS3_ERR_URI_TOO_LONG; } if (protocol_version == 1) { - snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s/%s%s", protocol, - domain, + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s%s/%s%s", protocol, + domain, port_str, bucket, object); } else { - snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s.%s%s", protocol, - bucket, domain, + snprintf(uri_buffer, MAX_URI_LENGTH - 1, "%s://%s.%s%s%s", protocol, + bucket, domain, port_str, object); } } @@ -717,8 +725,8 @@ uint8_t execute_request(ms3_st *ms3, command_t cmd, const char *bucket, ms3->query_buffer); } - res = build_request_uri(curl, ms3->base_domain, bucket, path, query, - ms3->use_http, ms3->protocol_version); + res = build_request_uri(curl, ms3->base_domain, ms3->port, bucket, path, + query, ms3->use_http, ms3->protocol_version); if (res) { diff --git a/src/structs.h b/src/structs.h index 8d30a57..de80b53 100644 --- a/src/structs.h +++ b/src/structs.h @@ -42,6 +42,7 @@ struct ms3_st char *s3secret; char *region; char *base_domain; + int port; // 0 means "Use default" size_t buffer_chunk_size; CURL *curl; char *last_error;

1 0