revision-id: 72055d6941d9e39e2b3672d2423811ce5ffde587 (mariadb-10.2.2-870-g72055d6941d) parent(s): 2facb2d39b1ad46f3a739be5043e13ef94592dd4 author: Sergei Petrunia committer: Sergei Petrunia timestamp: 2019-09-16 14:42:24 +0300 message: MDEV-20371: Invalid reads at plan refinement stage: join->positions... (Backported to 10.2-compatiblity, build 22257) best_access_path() is called from two optimization phases: 1. Plan choice phase, in choose_plan(). Here, the join prefix being considered is in join->positions[] 2. Plan refinement stage, in fix_semijoin_strategies_for_picked_join_order Here, the join prefix is in join->best_positions[] It used to access join->positions[] from stage #2. This didnt cause any valgrind or asan failures (as join->positions[] has been written-to before) but the effect was similar to that of reading the random data: The join prefix we've picked (in join->best_positions) could have nothing in common with the join prefix that was last to be considered (in join->positions). --- sql/opt_subselect.cc | 26 ++++++++++++++++---------- sql/opt_subselect.h | 8 ++++++-- sql/sql_select.cc | 38 +++++++++++++++++++++++--------------- sql/sql_select.h | 8 +++++++- 4 files changed, 52 insertions(+), 28 deletions(-) diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc index b4350551461..522aa299658 100644 --- a/sql/opt_subselect.cc +++ b/sql/opt_subselect.cc @@ -454,11 +454,6 @@ bool find_eq_ref_candidate(TABLE *table, table_map sj_inner_tables); static SJ_MATERIALIZATION_INFO * at_sjmat_pos(const JOIN *join, table_map remaining_tables, const JOIN_TAB *tab, uint idx, bool *loose_scan); -void best_access_path(JOIN *join, JOIN_TAB *s, - table_map remaining_tables, uint idx, - bool disable_jbuf, double record_count, - POSITION *pos, POSITION *loose_scan_pos); - static Item *create_subq_in_equalities(THD *thd, SJ_MATERIALIZATION_INFO *sjm, Item_in_subselect *subq_pred); static void remove_sj_conds(THD *thd, Item **tree); @@ -2659,6 +2654,13 @@ void advance_sj_state(JOIN *join, table_map remaining_tables, uint idx, NULL, }; +#ifdef HAVE_valgrind + new (&pos->firstmatch_picker) Firstmatch_picker; + new (&pos->loosescan_picker) LooseScan_picker; + new (&pos->sjmat_picker) Sj_materialization_picker; + new (&pos->dups_weedout_picker) Duplicate_weedout_picker; +#endif + if (join->emb_sjm_nest) { /* @@ -2942,7 +2944,8 @@ bool Sj_materialization_picker::check_qep(JOIN *join, bool disable_jbuf= (join->thd->variables.join_cache_level == 0); for (i= first_tab + mat_info->tables; i <= idx; i++) { - best_access_path(join, join->positions[i].table, rem_tables, i, + best_access_path(join, join->positions[i].table, rem_tables, + join->positions, i, disable_jbuf, prefix_rec_count, &curpos, &dummy); prefix_rec_count *= curpos.records_read; prefix_cost += curpos.read_time; @@ -3555,7 +3558,8 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join) join->cur_sj_inner_tables= 0; for (i= first + sjm->tables; i <= tablenr; i++) { - best_access_path(join, join->best_positions[i].table, rem_tables, i, + best_access_path(join, join->best_positions[i].table, rem_tables, + join->best_positions, i, FALSE, prefix_rec_count, join->best_positions + i, &dummy); prefix_rec_count *= join->best_positions[i].records_read; @@ -3587,8 +3591,9 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join) { if (join->best_positions[idx].use_join_buffer) { - best_access_path(join, join->best_positions[idx].table, - rem_tables, idx, TRUE /* no jbuf */, + best_access_path(join, join->best_positions[idx].table, + rem_tables, join->best_positions, idx, + TRUE /* no jbuf */, record_count, join->best_positions + idx, &dummy); } record_count *= join->best_positions[idx].records_read; @@ -3618,7 +3623,8 @@ void fix_semijoin_strategies_for_picked_join_order(JOIN *join) if (join->best_positions[idx].use_join_buffer || (idx == first)) { best_access_path(join, join->best_positions[idx].table, - rem_tables, idx, TRUE /* no jbuf */, + rem_tables, join->best_positions, idx, + TRUE /* no jbuf */, record_count, join->best_positions + idx, &loose_scan_pos); if (idx==first) diff --git a/sql/opt_subselect.h b/sql/opt_subselect.h index 7954becfad4..65d08bef2eb 100644 --- a/sql/opt_subselect.h +++ b/sql/opt_subselect.h @@ -88,6 +88,7 @@ class Loose_scan_opt KEYUSE *best_loose_scan_start_key; uint best_max_loose_keypart; + table_map best_ref_depend_map; public: Loose_scan_opt(): @@ -250,13 +251,14 @@ class Loose_scan_opt best_loose_scan_records= records; best_max_loose_keypart= max_loose_keypart; best_loose_scan_start_key= start_key; + best_ref_depend_map= 0; } } } } void check_ref_access_part2(uint key, KEYUSE *start_key, double records, - double read_time) + double read_time, table_map ref_depend_map_arg) { if (part1_conds_met && read_time < best_loose_scan_cost) { @@ -266,6 +268,7 @@ class Loose_scan_opt best_loose_scan_records= records; best_max_loose_keypart= max_loose_keypart; best_loose_scan_start_key= start_key; + best_ref_depend_map= ref_depend_map_arg; } } @@ -281,6 +284,7 @@ class Loose_scan_opt best_loose_scan_records= rows2double(quick->records); best_max_loose_keypart= quick_max_loose_keypart; best_loose_scan_start_key= NULL; + best_ref_depend_map= 0; } } @@ -296,7 +300,7 @@ class Loose_scan_opt pos->loosescan_picker.loosescan_parts= best_max_loose_keypart + 1; pos->use_join_buffer= FALSE; pos->table= tab; - // todo need ref_depend_map ? + pos->ref_depend_map= best_ref_depend_map; DBUG_PRINT("info", ("Produced a LooseScan plan, key %s, %s", tab->table->key_info[best_loose_scan_key].name, best_loose_scan_start_key? "(ref access)": diff --git a/sql/sql_select.cc b/sql/sql_select.cc index be2aeb52618..9bc47569f1a 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -97,10 +97,6 @@ static int sort_keyuse(KEYUSE *a,KEYUSE *b); static bool are_tables_local(JOIN_TAB *jtab, table_map used_tables); static bool create_ref_for_key(JOIN *join, JOIN_TAB *j, KEYUSE *org_keyuse, bool allow_full_scan, table_map used_tables); -void best_access_path(JOIN *join, JOIN_TAB *s, - table_map remaining_tables, uint idx, - bool disable_jbuf, double record_count, - POSITION *pos, POSITION *loose_scan_pos); static void optimize_straight_join(JOIN *join, table_map join_tables); static bool greedy_search(JOIN *join, table_map remaining_tables, uint depth, uint prune_level, @@ -4573,6 +4569,13 @@ make_join_statistics(JOIN *join, List<TABLE_LIST> &tables_list, { if (choose_plan(join, all_table_map & ~join->const_table_map)) goto error; + +#ifdef HAVE_valgrind + // JOIN::positions holds the current query plan. We've already + // made the plan choice, so we should only use JOIN::best_positions + for (uint k=join->const_tables; k < join->table_count; k++) + MEM_UNDEFINED(&join->positions[k], sizeof(join->positions[k])); +#endif } else { @@ -6288,6 +6291,7 @@ void best_access_path(JOIN *join, JOIN_TAB *s, table_map remaining_tables, + const POSITION *join_positions, uint idx, bool disable_jbuf, double record_count, @@ -6391,7 +6395,7 @@ best_access_path(JOIN *join, if (!(keyuse->used_tables & ~join->const_table_map)) const_part|= keyuse->keypart_map; - double tmp2= prev_record_reads(join->positions, idx, + double tmp2= prev_record_reads(join_positions, idx, (found_ref | keyuse->used_tables)); if (tmp2 < best_prev_record_reads) { @@ -6432,7 +6436,7 @@ best_access_path(JOIN *join, Really, there should be records=0.0 (yes!) but 1.0 would be probably safer */ - tmp= prev_record_reads(join->positions, idx, found_ref); + tmp= prev_record_reads(join_positions, idx, found_ref); records= 1.0; } else @@ -6448,7 +6452,7 @@ best_access_path(JOIN *join, if ((key_flags & (HA_NOSAME | HA_NULL_PART_KEY)) == HA_NOSAME || MY_TEST(key_flags & HA_EXT_NOSAME)) { - tmp = prev_record_reads(join->positions, idx, found_ref); + tmp = prev_record_reads(join_positions, idx, found_ref); records=1.0; } else @@ -6692,7 +6696,8 @@ best_access_path(JOIN *join, } tmp += s->startup_cost; - loose_scan_opt.check_ref_access_part2(key, start_key, records, tmp); + loose_scan_opt.check_ref_access_part2(key, start_key, records, tmp, + found_ref); } /* not ft_key */ if (tmp + 0.0001 < best_time - records/(double) TIME_FOR_COMPARE) { @@ -7364,7 +7369,8 @@ optimize_straight_join(JOIN *join, table_map join_tables) for (JOIN_TAB **pos= join->best_ref + idx ; (s= *pos) ; pos++) { /* Find the best access method from 's' to the current partial plan */ - best_access_path(join, s, join_tables, idx, disable_jbuf, record_count, + best_access_path(join, s, join_tables, join->positions, idx, + disable_jbuf, record_count, join->positions + idx, &loose_scan_pos); /* compute the cost of the new plan extended with 's' */ @@ -8270,8 +8276,9 @@ best_extension_by_limited_search(JOIN *join, /* Find the best access method from 's' to the current partial plan */ POSITION loose_scan_pos; - best_access_path(join, s, remaining_tables, idx, disable_jbuf, - record_count, join->positions + idx, &loose_scan_pos); + best_access_path(join, s, remaining_tables, join->positions, idx, + disable_jbuf, record_count, join->positions + idx, + &loose_scan_pos); /* Compute the cost of extending the plan with 's', avoid overflow */ if (position->records_read < DBL_MAX / record_count) @@ -8654,11 +8661,11 @@ cache_record_length(JOIN *join,uint idx) */ double -prev_record_reads(POSITION *positions, uint idx, table_map found_ref) +prev_record_reads(const POSITION *positions, uint idx, table_map found_ref) { double found=1.0; - POSITION *pos_end= positions - 1; - for (POSITION *pos= positions + idx - 1; pos != pos_end; pos--) + const POSITION *pos_end= positions - 1; + for (const POSITION *pos= positions + idx - 1; pos != pos_end; pos--) { if (pos->table->table->map & found_ref) { @@ -15560,7 +15567,8 @@ void optimize_wo_join_buffering(JOIN *join, uint first_tab, uint last_tab, if ((i == first_tab && first_alt) || join->positions[i].use_join_buffer) { /* Find the best access method that would not use join buffering */ - best_access_path(join, rs, reopt_remaining_tables, i, + best_access_path(join, rs, reopt_remaining_tables, + join->positions, i, TRUE, rec_count, &pos, &loose_scan_pos); } diff --git a/sql/sql_select.h b/sql/sql_select.h index eb78f4216f9..98fbb1f4d5b 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -794,6 +794,7 @@ class LooseScan_picker : public Semi_join_strategy_picker friend void best_access_path(JOIN *join, JOIN_TAB *s, table_map remaining_tables, + const struct st_position *join_positions, uint idx, bool disable_jbuf, double record_count, @@ -1970,6 +1971,11 @@ class store_key_const_item :public store_key_item } }; +void best_access_path(JOIN *join, JOIN_TAB *s, + table_map remaining_tables, + const POSITION *join_positions, uint idx, + bool disable_jbuf, double record_count, + POSITION *pos, POSITION *loose_scan_pos); bool cp_buffer_from_ref(THD *thd, TABLE *table, TABLE_REF *ref); bool error_if_full_join(JOIN *join); int report_error(TABLE *table, int error); @@ -2334,7 +2340,7 @@ bool instantiate_tmp_table(TABLE *table, KEY *keyinfo, ulonglong options); bool open_tmp_table(TABLE *table); void setup_tmp_table_column_bitmaps(TABLE *table, uchar *bitmaps); -double prev_record_reads(POSITION *positions, uint idx, table_map found_ref); +double prev_record_reads(const POSITION *positions, uint idx, table_map found_ref); void fix_list_after_tbl_changes(SELECT_LEX *new_parent, List<TABLE_LIST> *tlist); struct st_cond_statistic