revision-id: 1966b71bfb250d551d53daf7f6606337900bbb0f (mariadb-10.4.7-64-g1966b71bfb2) parent(s): 5c5452a5a086a9584efb2255059da671fff6e484 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-09-12 19:29:07 +0300 message: MDEV-20371: Invalid reads at plan refinement stage: join->positions... (re-committing in 10.4) best_access_path() is called from two optimization phases: 1. Plan choice phase, in choose_plan(). Here, the join prefix being considered is in join->positions[] 2. Plan refinement stage, in fix_semijoin_strategies_for_picked_join_order Here, the join prefix is in join->best_positions[] It used to access join->positions[] from stage #2. This didnt cause any valgrind or asan failures (as join->positions[] has been written-to before) but the effect was similar to that of reading the random data: The join prefix we've picked (in join->best_positions) could have nothing in common with the join prefix that was last to be considered (in join->positions). --- sql/opt_subselect.cc | 26 +++++++++++++++++--------- sql/opt_subselect.h | 8 ++++++-- sql/sql_select.cc | 37 ++++++++++++++++++++++--------------- sql/sql_select.h | 9 ++++++++- 4 files changed, 53 insertions(+), 27 deletions(-) diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc index c4cb9b81170..22367ef85e4 100644 --- a/sql/opt_subselect.cc +++ b/sql/opt_subselect.cc @@ -453,10 +453,6 @@ bool find_eq_ref_candidate(TABLE *table, table_map sj_inner_tables); static SJ_MATERIALIZATION_INFO * at_sjmat_pos(const JOIN *join, table_map remaining_tables, const JOIN_TAB *tab, uint idx, bool *loose_scan); -void best_access_path(JOIN *join, JOIN_TAB *s, - table_map remaining_tables, uint idx, - bool disable_jbuf, double record_count, - POSITION *pos, POSITION *loose_scan_pos); static Item *create_subq_in_equalities(THD *thd, SJ_MATERIALIZATION_INFO *sjm, Item_in_subselect *subq_pred); @@ -2804,6 +2800,14 @@ void advance_sj_state(JOIN *join, table_map remaining_tables, uint idx, &pos->dups_weedout_picker, NULL, }; + +#ifdef HAVE_valgrind + new (&pos->firstmatch_picker) Firstmatch_picker; + new (&pos->loosescan_picker) LooseScan_picker; + new (&pos->sjmat_picker) Sj_materialization_picker; + new (&pos->dups_weedout_picker) Duplicate_weedout_picker; +#endif + Json_writer_array trace_steps(join->thd, "semijoin_strategy_choice"); /* Update join->cur_sj_inner_tables (Used by FirstMatch in this function and @@ -3121,7 +3125,8 @@ bool Sj_materialization_picker::check_qep(JOIN *join, Json_writer_temp_disable trace_semijoin_mat_scan(thd); for (i= first_tab + mat_info->tables; i <= idx; i++) { - best_access_path(join, join->positions[i].table, rem_tables, i, + best_access_path(join, join->positions[i].table, rem_tables, + join->positions, i, disable_jbuf, prefix_rec_count, &curpos, &dummy); prefix_rec_count= COST_MULT(prefix_rec_count, curpos.records_read); prefix_cost= COST_ADD(prefix_cost, curpos.read_time); @@ -3790,7 +3795,8 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join) Json_writer_object trace_one_table(thd); trace_one_table.add_table_name(join->best_positions[i].table); } - best_access_path(join, join->best_positions[i].table, rem_tables, i, + best_access_path(join, join->best_positions[i].table, rem_tables, + join->best_positions, i, FALSE, prefix_rec_count, join->best_positions + i, &dummy); prefix_rec_count *= join->best_positions[i].records_read; @@ -3830,8 +3836,9 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join) } if (join->best_positions[idx].use_join_buffer) { - best_access_path(join, join->best_positions[idx].table, - rem_tables, idx, TRUE /* no jbuf */, + best_access_path(join, join->best_positions[idx].table, + rem_tables, join->best_positions, idx, + TRUE /* no jbuf */, record_count, join->best_positions + idx, &dummy); } record_count *= join->best_positions[idx].records_read; @@ -3869,7 +3876,8 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join) if (join->best_positions[idx].use_join_buffer || (idx == first)) { best_access_path(join, join->best_positions[idx].table, - rem_tables, idx, TRUE /* no jbuf */, + rem_tables, join->best_positions, idx, + TRUE /* no jbuf */, record_count, join->best_positions + idx, &loose_scan_pos); if (idx==first) diff --git a/sql/opt_subselect.h b/sql/opt_subselect.h index 6210fc972c8..d1dd8e3e135 100644 --- a/sql/opt_subselect.h +++ b/sql/opt_subselect.h @@ -91,6 +91,7 @@ class Loose_scan_opt KEYUSE *best_loose_scan_start_key; uint best_max_loose_keypart; + table_map best_ref_depend_map; public: Loose_scan_opt(): @@ -252,13 +253,14 @@ class Loose_scan_opt best_loose_scan_records= records; best_max_loose_keypart= max_loose_keypart; best_loose_scan_start_key= start_key; + best_ref_depend_map= 0; } } } } void check_ref_access_part2(uint key, KEYUSE *start_key, double records, - double read_time) + double read_time, table_map ref_depend_map_arg) { if (part1_conds_met && read_time < best_loose_scan_cost) { @@ -268,6 +270,7 @@ class Loose_scan_opt best_loose_scan_records= records; best_max_loose_keypart= max_loose_keypart; best_loose_scan_start_key= start_key; + best_ref_depend_map= ref_depend_map_arg; } } @@ -283,6 +286,7 @@ class Loose_scan_opt best_loose_scan_records= rows2double(quick->records); best_max_loose_keypart= quick_max_loose_keypart; best_loose_scan_start_key= NULL; + best_ref_depend_map= 0; } } @@ -299,7 +303,7 @@ class Loose_scan_opt pos->use_join_buffer= FALSE; pos->table= tab; pos->range_rowid_filter_info= tab->range_rowid_filter_info; - // todo need ref_depend_map ? + pos->ref_depend_map= best_ref_depend_map; DBUG_PRINT("info", ("Produced a LooseScan plan, key %s, %s", tab->table->key_info[best_loose_scan_key].name.str, best_loose_scan_start_key? "(ref access)": diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 3dae58a78e2..641a398cbc6 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -107,10 +107,6 @@ static bool create_ref_for_key(JOIN *join, JOIN_TAB *j, KEYUSE *org_keyuse, static ha_rows get_quick_record_count(THD *thd, SQL_SELECT *select, TABLE *table, const key_map *keys,ha_rows limit); -void best_access_path(JOIN *join, JOIN_TAB *s, - table_map remaining_tables, uint idx, - bool disable_jbuf, double record_count, - POSITION *pos, POSITION *loose_scan_pos); static void optimize_straight_join(JOIN *join, table_map join_tables); static bool greedy_search(JOIN *join, table_map remaining_tables, uint depth, uint prune_level, @@ -5479,6 +5475,13 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list, { if (choose_plan(join, all_table_map & ~join->const_table_map)) goto error; + +#ifdef HAVE_valgrind + // JOIN::positions holds the current query plan. We've already + // made the plan choice, so we should only use JOIN::best_positions + for (uint k=join->const_tables; k < join->table_count; k++) + MEM_UNDEFINED(&join->positions[k], sizeof(join->positions[k])); +#endif } else { @@ -7178,6 +7181,7 @@ void best_access_path(JOIN *join, JOIN_TAB *s, table_map remaining_tables, + const POSITION *join_positions, uint idx, bool disable_jbuf, double record_count, @@ -7298,7 +7302,7 @@ best_access_path(JOIN *join, if (!keyuse->val->maybe_null || keyuse->null_rejecting) notnull_part|=keyuse->keypart_map; - double tmp2= prev_record_reads(join->positions, idx, + double tmp2= prev_record_reads(join_positions, idx, (found_ref | keyuse->used_tables)); if (tmp2 < best_prev_record_reads) { @@ -7340,7 +7344,7 @@ best_access_path(JOIN *join, Really, there should be records=0.0 (yes!) but 1.0 would be probably safer */ - tmp= prev_record_reads(join->positions, idx, found_ref); + tmp= prev_record_reads(join_positions, idx, found_ref); records= 1.0; type= JT_FT; trace_access_idx.add("access_type", join_type_str[type]) @@ -7369,7 +7373,7 @@ best_access_path(JOIN *join, type= JT_EQ_REF; trace_access_idx.add("access_type", join_type_str[type]) .add("index", keyinfo->name); - tmp = prev_record_reads(join->positions, idx, found_ref); + tmp = prev_record_reads(join_positions, idx, found_ref); records=1.0; } else @@ -7657,7 +7661,8 @@ best_access_path(JOIN *join, } tmp= COST_ADD(tmp, s->startup_cost); - loose_scan_opt.check_ref_access_part2(key, start_key, records, tmp); + loose_scan_opt.check_ref_access_part2(key, start_key, records, tmp, + found_ref); } /* not ft_key */ if (records < DBL_MAX) @@ -8447,7 +8452,8 @@ optimize_straight_join(JOIN *join, table_map join_tables) trace_one_table.add_table_name(s); } /* Find the best access method from 's' to the current partial plan */ - best_access_path(join, s, join_tables, idx, disable_jbuf, record_count, + best_access_path(join, s, join_tables, join->positions, idx, + disable_jbuf, record_count, position, &loose_scan_pos); /* compute the cost of the new plan extended with 's' */ @@ -9376,8 +9382,8 @@ best_extension_by_limited_search(JOIN *join, /* Find the best access method from 's' to the current partial plan */ POSITION loose_scan_pos; - best_access_path(join, s, remaining_tables, idx, disable_jbuf, - record_count, position, &loose_scan_pos); + best_access_path(join, s, remaining_tables, join->positions, idx, + disable_jbuf, record_count, position, &loose_scan_pos); /* Compute the cost of extending the plan with 's' */ current_record_count= COST_MULT(record_count, position->records_read); @@ -9781,11 +9787,11 @@ cache_record_length(JOIN *join,uint idx) */ double -prev_record_reads(POSITION *positions, uint idx, table_map found_ref) +prev_record_reads(const POSITION *positions, uint idx, table_map found_ref) { double found=1.0; - POSITION *pos_end= positions - 1; - for (POSITION *pos= positions + idx - 1; pos != pos_end; pos--) + const POSITION *pos_end= positions - 1; + for (const POSITION *pos= positions + idx - 1; pos != pos_end; pos--) { if (pos->table->table->map & found_ref) { @@ -16675,7 +16681,8 @@ void optimize_wo_join_buffering(JOIN *join, uint first_tab, uint last_tab, if ((i == first_tab && first_alt) || join->positions[i].use_join_buffer) { /* Find the best access method that would not use join buffering */ - best_access_path(join, rs, reopt_remaining_tables, i, + best_access_path(join, rs, reopt_remaining_tables, + join->positions, i, TRUE, rec_count, &pos, &loose_scan_pos); } diff --git a/sql/sql_select.h b/sql/sql_select.h index 4ec258f3653..b6359307215 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -854,6 +854,7 @@ class LooseScan_picker : public Semi_join_strategy_picker friend void best_access_path(JOIN *join, JOIN_TAB *s, table_map remaining_tables, + const struct st_position *join_positions, uint idx, bool disable_jbuf, double record_count, @@ -2071,6 +2072,12 @@ class store_key_const_item :public store_key_item } }; +void best_access_path(JOIN *join, JOIN_TAB *s, + table_map remaining_tables, + const POSITION *join_positions, uint idx, + bool disable_jbuf, double record_count, + POSITION *pos, POSITION *loose_scan_pos); + bool cp_buffer_from_ref(THD *thd, TABLE *table, TABLE_REF *ref); bool error_if_full_join(JOIN *join); int report_error(TABLE *table, int error); @@ -2435,7 +2442,7 @@ bool instantiate_tmp_table(TABLE *table, KEY *keyinfo, ulonglong options); bool open_tmp_table(TABLE *table); void setup_tmp_table_column_bitmaps(TABLE *table, uchar *bitmaps); -double prev_record_reads(POSITION *positions, uint idx, table_map found_ref); +double prev_record_reads(const POSITION *positions, uint idx, table_map found_ref); void fix_list_after_tbl_changes(SELECT_LEX *new_parent, List<TABLE_LIST> *tlist); double get_tmp_table_lookup_cost(THD *thd, double row_count, uint row_size); double get_tmp_table_write_cost(THD *thd, double row_count, uint row_size);