- developers - lists.mariadb.org

[Maria-developers] Rev 2725: Apply Yasufumi's patch for MBUG#417751. in file:///home/psergey/bzr-new/mysql-5.1-maria-contd2/
by Sergey Petrunya 10 Sep '09

10 Sep '09

At file:///home/psergey/bzr-new/mysql-5.1-maria-contd2/ ------------------------------------------------------------ revno: 2725 revision-id: psergey(a)askmonty.org-20090909221538-1k8a5q1qzqdyymyw parent: psergey(a)askmonty.org-20090909215928-jd9v62cem7e2uukw committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: mysql-5.1-maria-contd2 timestamp: Thu 2009-09-10 02:15:38 +0400 message: Apply Yasufumi's patch for MBUG#417751. === modified file 'storage/xtradb/btr/btr0cur.c' --- a/storage/xtradb/btr/btr0cur.c 2009-06-25 01:43:25 +0000 +++ b/storage/xtradb/btr/btr0cur.c 2009-09-09 22:15:38 +0000 @@ -3267,6 +3267,7 @@ } while (rec != supremum) { + rec_t* next_rec; /* count recs */ if (stats_method == SRV_STATS_METHOD_IGNORE_NULLS) { n_recs++; @@ -3280,8 +3281,7 @@ n_not_nulls[j]++; } } - - rec_t* next_rec = page_rec_get_next(rec); + next_rec = page_rec_get_next(rec); if (next_rec == supremum) { break; } === modified file 'storage/xtradb/buf/buf0buddy.c' --- a/storage/xtradb/buf/buf0buddy.c 2009-06-25 01:43:25 +0000 +++ b/storage/xtradb/buf/buf0buddy.c 2009-09-09 22:15:38 +0000 @@ -461,16 +461,15 @@ actually is a properly initialized buf_page_t object. */ if (size >= PAGE_ZIP_MIN_SIZE) { - if (!have_page_hash_mutex) - mutex_exit(&zip_free_mutex); - /* This is a compressed page. */ mutex_t* mutex; if (!have_page_hash_mutex) { + mutex_exit(&zip_free_mutex); mutex_enter(&LRU_list_mutex); rw_lock_x_lock(&page_hash_latch); } + /* The src block may be split into smaller blocks, some of which may be free. Thus, the mach_read_from_4() calls below may attempt to read === modified file 'storage/xtradb/buf/buf0flu.c' --- a/storage/xtradb/buf/buf0flu.c 2009-08-09 07:04:58 +0000 +++ b/storage/xtradb/buf/buf0flu.c 2009-09-09 22:15:38 +0000 @@ -1233,13 +1233,13 @@ + BUF_FLUSH_EXTRA_MARGIN) && (distance < BUF_LRU_FREE_SEARCH_LEN)) { + mutex_t* block_mutex; if (!bpage->in_LRU_list) { /* reatart. but it is very optimistic */ bpage = UT_LIST_GET_LAST(buf_pool->LRU); continue; } - - mutex_t* block_mutex = buf_page_get_mutex(bpage); + block_mutex = buf_page_get_mutex_enter(bpage); retry_lock: mutex_enter(block_mutex); === modified file 'storage/xtradb/fil/fil0fil.c' --- a/storage/xtradb/fil/fil0fil.c 2009-06-25 01:43:25 +0000 +++ b/storage/xtradb/fil/fil0fil.c 2009-09-09 22:15:38 +0000 @@ -3039,6 +3039,13 @@ ulint i; int len; ib_uint64_t current_lsn; + ulint size_low, size_high, size; + ib_int64_t size_bytes; + dict_table_t* table; + dict_index_t* index; + fil_system_t* system; + fil_node_t* node = NULL; + fil_space_t* space; current_lsn = log_get_lsn(); @@ -3060,25 +3067,20 @@ success = os_file_write(filepath, file, page, 0, 0, UNIV_PAGE_SIZE); /* get file size */ - ulint size_low, size_high, size; - ib_int64_t size_bytes; os_file_get_size(file, &size_low, &size_high); size_bytes = (((ib_int64_t)size_high) << 32) + (ib_int64_t)size_low; /* get cruster index information */ - dict_table_t* table; - dict_index_t* index; table = dict_table_get_low(name); index = dict_table_get_first_index(table); ut_a(index->page==3); - /* read metadata from .exp file */ n_index = 0; - bzero(old_id, sizeof(old_id)); - bzero(new_id, sizeof(new_id)); - bzero(root_page, sizeof(root_page)); + memset(old_id, 0, sizeof(old_id)); + memset(new_id, 0, sizeof(new_id)); + memset(root_page, 0, sizeof(root_page)); info_file_path = fil_make_ibd_name(name, FALSE); len = strlen(info_file_path); @@ -3128,10 +3130,10 @@ mem_heap_t* heap = NULL; ulint offsets_[REC_OFFS_NORMAL_SIZE]; ulint* offsets = offsets_; + ib_int64_t offset; + size = (ulint) (size_bytes / UNIV_PAGE_SIZE); /* over write space id of all pages */ - ib_int64_t offset; - rec_offs_init(offsets_); fprintf(stderr, "InnoDB: Progress in %:"); @@ -3284,10 +3286,8 @@ } mem_free(info_file_path); - fil_system_t* system = fil_system; + system = fil_system; mutex_enter(&(system->mutex)); - fil_node_t* node = NULL; - fil_space_t* space; space = fil_space_get_by_id(id); if (space) node = UT_LIST_GET_FIRST(space->chain); === modified file 'storage/xtradb/handler/i_s.cc' --- a/storage/xtradb/handler/i_s.cc 2009-06-25 01:43:25 +0000 +++ b/storage/xtradb/handler/i_s.cc 2009-09-09 22:15:38 +0000 @@ -806,7 +806,7 @@ field_store_string(table->field[0], NULL); p = (char *)index->table_name; } - strcpy(table_name_raw, p); + strcpy(table_name_raw, (const char*)p); filename_to_tablename(table_name_raw, table_name, sizeof(table_name)); field_store_string(table->field[1], table_name); field_store_string(table->field[2], index->name); @@ -2861,7 +2861,7 @@ } else { rec_per_key = n_rows; } - snprintf(buff, 256, (i == index->n_uniq)?"%llu":"%llu, ", + ut_snprintf(buff, 256, (i == index->n_uniq)?"%llu":"%llu, ", rec_per_key); strncat(row_per_keys, buff, 256 - strlen(row_per_keys)); } === modified file 'storage/xtradb/srv/srv0srv.c' --- a/storage/xtradb/srv/srv0srv.c 2009-09-08 16:04:58 +0000 +++ b/storage/xtradb/srv/srv0srv.c 2009-09-09 22:15:38 +0000 @@ -2694,11 +2694,11 @@ /* defence line (max_checkpoint_age * 1/2) */ ib_uint64_t lsn = log_sys->lsn; - mutex_exit(&(log_sys->mutex)); - ib_uint64_t level, bpl; buf_page_t* bpage; + mutex_exit(&(log_sys->mutex)); + mutex_enter(&flush_list_mutex); level = 0; === modified file 'storage/xtradb/srv/srv0start.c' --- a/storage/xtradb/srv/srv0start.c 2009-08-03 20:09:53 +0000 +++ b/storage/xtradb/srv/srv0start.c 2009-09-09 22:15:38 +0000 @@ -1134,7 +1134,12 @@ os_aio_use_native_aio = FALSE; } else { /* On Win 2000 and XP use async i/o */ - os_aio_use_native_aio = TRUE; + //os_aio_use_native_aio = TRUE; + os_aio_use_native_aio = FALSE; + fprintf(stderr, + "InnoDB: Windows native async i/o is disabled as default.\n" + "InnoDB: It is not applicable for the current" + " multi io threads implementation.\n"); } #endif if (srv_file_flush_method_str == NULL) { @@ -1170,6 +1175,12 @@ } else if (0 == ut_strcmp(srv_file_flush_method_str, "async_unbuffered")) { srv_win_file_flush_method = SRV_WIN_IO_UNBUFFERED; + os_aio_use_native_aio = TRUE; + srv_n_read_io_threads = srv_n_write_io_threads = 1; + fprintf(stderr, + "InnoDB: 'async_unbuffered' was detected as innodb_flush_method.\n" + "InnoDB: Windows native async i/o is enabled.\n" + "InnoDB: And io threads are restricted.\n"); #endif } else { fprintf(stderr,

1 0

[Maria-developers] Rev 2724: Merge MySQL 5.1 -> MariaDB in file:///home/psergey/bzr-new/mysql-5.1-maria-contd2/
by Sergey Petrunya 09 Sep '09

09 Sep '09

At file:///home/psergey/bzr-new/mysql-5.1-maria-contd2/ ------------------------------------------------------------ revno: 2724 revision-id: psergey(a)askmonty.org-20090909215928-jd9v62cem7e2uukw parent: psergey(a)askmonty.org-20090909210657-i1of5o03ymoozwvs committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: mysql-5.1-maria-contd2 timestamp: Thu 2009-09-10 01:59:28 +0400 message: Merge MySQL 5.1 -> MariaDB - Don't follow mainline on use of UNINIT_VAR yet, we haven't merged it in. === modified file 'sql/opt_range.cc' --- a/sql/opt_range.cc 2009-09-09 21:06:57 +0000 +++ b/sql/opt_range.cc 2009-09-09 21:59:28 +0000 @@ -2243,7 +2243,7 @@ KEY *key_info; PARAM param; - if (check_stack_overrun(thd, 2*STACK_MIN_SIZE + sizeof(PARAM), buff)) + if (check_stack_overrun(thd, 2*STACK_MIN_SIZE, buff)) DBUG_RETURN(0); // Fatal error flag is set /* set up parameter that is passed to all functions */ @@ -3760,8 +3760,9 @@ DBUG_PRINT("info", ("index_merge scans cost %g", imerge_cost)); if (imerge_too_expensive || (imerge_cost > read_time) || - ((non_cpk_scan_records+cpk_scan_records >= param->table->file->stats.records) && - read_time != DBL_MAX)) + ((non_cpk_scan_records+cpk_scan_records >= + param->table->file->stats.records) && + read_time != DBL_MAX)) { /* Bail out if it is obvious that both index_merge and ROR-union will be @@ -4838,11 +4839,11 @@ { int idx; SEL_ARG **key,**end, **key_to_read= NULL; - ha_rows UNINIT_VAR(best_records); /* protected by key_to_read */ + ha_rows best_records; TRP_RANGE* read_plan= NULL; bool pk_is_clustered= param->table->file->primary_key_is_clustered(); DBUG_ENTER("get_key_scans_params"); - + LINT_INIT(best_records); /* protected by key_to_read */ /* Note that there may be trees that have type SEL_TREE::KEY but contain no key reads at all, e.g. tree for expression "key1 is not null" where key1 @@ -6439,6 +6440,13 @@ return 0; // Can't optimize this } + if ((key1->min_flag | key2->min_flag) & GEOM_FLAG) + { + key1->free_tree(); + key2->free_tree(); + return 0; // Can't optimize this + } + key1->use_count--; key2->use_count--; SEL_ARG *e1=key1->first(), *e2=key2->first(), *new_tree=0; @@ -6789,7 +6797,9 @@ SEL_ARG * SEL_ARG::insert(SEL_ARG *key) { - SEL_ARG *element,**UNINIT_VAR(par),*UNINIT_VAR(last_element); + SEL_ARG *element,**par,*last_element; + LINT_INIT(par); + LINT_INIT(last_element); for (element= this; element != &null_element ; ) {

1 0

[Maria-developers] Rev 2863: Re-commit BUG#42742 in file:///home/psergey/dev/mysql-next-subqbugs/
by Sergey Petrunya 09 Sep '09

09 Sep '09

At file:///home/psergey/dev/mysql-next-subqbugs/ ------------------------------------------------------------ revno: 2863 revision-id: psergey(a)askmonty.org-20090909185418-aqe8vozjy1zo037q parent: alik(a)sun.com-20090828100112-r73xkx8dhekz5bbb committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: mysql-next-subqbugs timestamp: Wed 2009-09-09 22:54:18 +0400 message: Re-commit BUG#42742 === modified file 'mysql-test/r/subselect_sj2.result' --- a/mysql-test/r/subselect_sj2.result 2009-03-21 15:31:38 +0000 +++ b/mysql-test/r/subselect_sj2.result 2009-09-09 18:54:18 +0000 @@ -689,3 +689,19 @@ id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables drop table t1, t2; +# +# BUG#42742: crash in setup_sj_materialization, Copy_field::set +# +create table t3 ( c1 year) engine=innodb; +insert into t3 values (2135),(2142); +create table t2 (c1 tinytext,c2 text,c6 timestamp) engine=innodb; +# The following must not crash, EXPLAIN should show one SJ strategy, not a mix: +explain select 1 from t2 where +c2 in (select 1 from t3, t2) and +c1 in (select convert(c6,char(1)) from t2); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 1 Using where +1 PRIMARY t2 ALL NULL NULL NULL NULL 1 +1 PRIMARY t2 ALL NULL NULL NULL NULL 1 Using where; Using join buffer +1 PRIMARY t3 ALL NULL NULL NULL NULL 2 FirstMatch(t2); Using join buffer +drop table t2, t3; === modified file 'mysql-test/r/subselect_sj2_jcl6.result' --- a/mysql-test/r/subselect_sj2_jcl6.result 2009-06-19 09:12:06 +0000 +++ b/mysql-test/r/subselect_sj2_jcl6.result 2009-09-09 18:54:18 +0000 @@ -693,6 +693,22 @@ id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables drop table t1, t2; +# +# BUG#42742: crash in setup_sj_materialization, Copy_field::set +# +create table t3 ( c1 year) engine=innodb; +insert into t3 values (2135),(2142); +create table t2 (c1 tinytext,c2 text,c6 timestamp) engine=innodb; +# The following must not crash, EXPLAIN should show one SJ strategy, not a mix: +explain select 1 from t2 where +c2 in (select 1 from t3, t2) and +c1 in (select convert(c6,char(1)) from t2); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 1 Using where +1 PRIMARY t2 ALL NULL NULL NULL NULL 1 Using join buffer +1 PRIMARY t2 ALL NULL NULL NULL NULL 1 Using where; Using join buffer +1 PRIMARY t3 ALL NULL NULL NULL NULL 2 FirstMatch(t2); Using join buffer +drop table t2, t3; set join_cache_level=default; show variables like 'join_cache_level'; Variable_name Value === modified file 'mysql-test/t/subselect_sj2.test' --- a/mysql-test/t/subselect_sj2.test 2009-03-21 15:31:38 +0000 +++ b/mysql-test/t/subselect_sj2.test 2009-09-09 18:54:18 +0000 @@ -872,3 +872,15 @@ explain select 1 from t2 where c2 = any (select log10(null) from t1 where c6 <null) ; drop table t1, t2; +--echo # +--echo # BUG#42742: crash in setup_sj_materialization, Copy_field::set +--echo # +create table t3 ( c1 year) engine=innodb; +insert into t3 values (2135),(2142); +create table t2 (c1 tinytext,c2 text,c6 timestamp) engine=innodb; +-- echo # The following must not crash, EXPLAIN should show one SJ strategy, not a mix: +explain select 1 from t2 where + c2 in (select 1 from t3, t2) and + c1 in (select convert(c6,char(1)) from t2); +drop table t2, t3; + === modified file 'sql/sql_select.cc' --- a/sql/sql_select.cc 2009-07-28 14:16:37 +0000 +++ b/sql/sql_select.cc 2009-09-09 18:54:18 +0000 @@ -8006,7 +8006,11 @@ uint i_end= first + join->best_positions[first].n_sj_tables; for (uint i= first; i < i_end; i++) + { + if (i != first) + join->best_positions[i].sj_strategy= SJ_OPT_NONE; handled_tabs |= join->best_positions[i].table->table->map; + } if (tablenr != first) pos->sj_strategy= SJ_OPT_NONE;

1 0

[Maria-developers] [Branch ~maria-captains/maria/5.1] Rev 2731: Added a missing dependency to build maria lib
by noreply＠launchpad.net 09 Sep '09

09 Sep '09

------------------------------------------------------------ revno: 2731 committer: Igor Babaev <igor(a)mysql.com> branch nick: maria-5.1-fix-windows timestamp: Wed 2009-09-09 11:11:31 -0700 message: Added a missing dependency to build maria lib modified: storage/maria/CMakeLists.txt -- lp:maria https://code.launchpad.net/~maria-captains/maria/5.1 Your team Maria developers is subscribed to branch lp:maria. To unsubscribe from this branch go to https://code.launchpad.net/~maria-captains/maria/5.1/+edit-subscription.

1 0

[Maria-developers] Progress (by Knielsen): mysqlbinlog: remove undesired effects of format description binlog statement (50)
by worklog-noreply＠askmonty.org 09 Sep '09

09 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: mysqlbinlog: remove undesired effects of format description binlog statement CREATION DATE..: Mon, 17 Aug 2009, 14:12 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 50 (http://askmonty.org/worklog/?tid=50) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 10 ESTIMATE.......: 10 (hours remain) ORIG. ESTIMATE.: 10 PROGRESS NOTES: -=-=(Knielsen - Wed, 09 Sep 2009, 18:31)=-=- Discussed with MySQL developers on Bug#46640. Committed a suggested fix: https://lists.launchpad.net/maria-developers/msg00926.html Worked 8 hours and estimate 10 hours remain (original estimate increased by 18 hours). -=-=(Guest - Wed, 09 Sep 2009, 11:50)=-=- High Level Description modified. --- /tmp/wklog.50.old.11584 2009-09-09 11:50:12.000000000 +0300 +++ /tmp/wklog.50.new.11584 2009-09-09 11:50:12.000000000 +0300 @@ -5,7 +5,11 @@ This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. -2. When one applies "format description binlog statement" at the slave, it -will change the slave's server_id when applied. +2. When one executes a BINLOG statement (containing "format description binlog +statement" or other events), this will cause subsequent statements run in +the same session to be binlogged with the server id of the last event +executed, not with the real server id. This can cause problems like infinite +recursive application in a circular replication topology. This WL is to fix these issues. + -=-=(Knielsen - Tue, 08 Sep 2009, 15:07)=-=- Low Level Design modified. --- /tmp/wklog.50.old.19208 2009-09-08 15:07:09.000000000 +0300 +++ /tmp/wklog.50.new.19208 2009-09-08 15:07:09.000000000 +0300 @@ -1 +1,34 @@ +I think the fix for point 2 is to replace the call to +apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct +call to ev->apply_event(). However, I need to check two things to be sure this +is correct: + +1. The existing code does + + if (!ev->when) ev->when= my_time(0) + +Need to understand if this is needed/correct or not. + +2. The existing code does ev->update_pos(). I think this is redundant for +BINLOG statement (as it uses a fake rli structure), but I need to check to +make sure. + +Once this is done, point 1 may no longer be needed. The user can use +--base64-output=never when applying the output to a 5.0 server, and omit that +option when applying the output to a 5.1 server. There should be no need to +omit the format description event in the output when other BINLOG statements +for row events are present, as it no longer changes the server id. In fact the +format description event is required to be able to execute other BINLOG +statements, as its purpose is to define the binary format of the events +contained in these statements. + +Alternatively, we could implement that if the format description event of the +source binlog has server version < 5.1, and --base64-output=auto (default), +then the format description event is omitted (and should any BINLOG statement +be needed, unlikely as it is from a 5.0 server, we will need to throw an +error). + +The binlog format version for 5.0 and 5.1 is actually the same, hence the need +to look at server version to guess if the format description event can be +omitted. -=-=(Knielsen - Tue, 08 Sep 2009, 14:29)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.17531 2009-09-08 14:29:30.000000000 +0300 +++ /tmp/wklog.50.new.17531 2009-09-08 14:29:30.000000000 +0300 @@ -12,3 +12,37 @@ One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? +First, the problem is not the format description log event. In fact all log +events have their own value of server_id, and every event in the BINLOG +statement is executed with its own id as server_id. + +It seems it was introduced deliberately in the patch for Bug#32407. This patch +makes sql thread and BINLOG statement share code path for executing the binary +event, even though the bug is really about something different (outputting +format description event to allow proper execution of other BINLOG +statements). + +Nevertheless, I think using the server_id in the events of a BINLOG statement +is wrong: + +1. It is different behaviour from executing normal statement-based events not +written using BINLOG. + +2. It causes the possibility for infinite cycle of replication in a cyclic +replication setup, if the server ids in BINLOG are different from all other +server ids in the cycle. + +3. The functionality of applying events with original server id from the event +is still available by pointing the slave thread to the binlog. The +functionality of applying row-based binlog events with server id of the server +executing the BINLOG statements is not otherwise available. + +In fact most of the code from the slave thread that is now also run when +executing BINLOG statements is redundant, or does the wrong thing (like +updating binlog position). + +Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be +able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is +it to be able to simulate the effect of the slave sql thread? I think the +former is the most important one. + -=-=(Knielsen - Mon, 07 Sep 2009, 10:06)=-=- Research worklog, bugs, and existing server code. Worked 2 hours and estimate 0 hours remain (original estimate increased by 2 hours). -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- Dependency created: 39 now depends on 50 -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.11389 2009-08-17 14:13:05.000000000 +0300 +++ /tmp/wklog.50.new.11389 2009-08-17 14:13:05.000000000 +0300 @@ -1 +1,14 @@ +First item +---------- +AFAIU what needs to be done is: +1. record a source server version (it is in the first binlog event). +2. don't emit a BINLOG statement if the recorded version number is 5.0.x or + below. + +and we'll get a 5.0-applicable binlog. + +Second item +----------- +One question that one needs to sort out before disabling server_id change is +why it was put there in the first place? Should it be always removed? DESCRIPTION: According to complaints in BUG#46640: 1. mysqlbinlog will emit a 5.0-incompatible "format description binlog statement" even when reading a binary log that was produced by 5.0. This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. 2. When one executes a BINLOG statement (containing "format description binlog statement" or other events), this will cause subsequent statements run in the same session to be binlogged with the server id of the last event executed, not with the real server id. This can cause problems like infinite recursive application in a circular replication topology. This WL is to fix these issues. HIGH-LEVEL SPECIFICATION: First item ---------- AFAIU what needs to be done is: 1. record a source server version (it is in the first binlog event). 2. don't emit a BINLOG statement if the recorded version number is 5.0.x or below. and we'll get a 5.0-applicable binlog. Second item ----------- One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? First, the problem is not the format description log event. In fact all log events have their own value of server_id, and every event in the BINLOG statement is executed with its own id as server_id. It seems it was introduced deliberately in the patch for Bug#32407. This patch makes sql thread and BINLOG statement share code path for executing the binary event, even though the bug is really about something different (outputting format description event to allow proper execution of other BINLOG statements). Nevertheless, I think using the server_id in the events of a BINLOG statement is wrong: 1. It is different behaviour from executing normal statement-based events not written using BINLOG. 2. It causes the possibility for infinite cycle of replication in a cyclic replication setup, if the server ids in BINLOG are different from all other server ids in the cycle. 3. The functionality of applying events with original server id from the event is still available by pointing the slave thread to the binlog. The functionality of applying row-based binlog events with server id of the server executing the BINLOG statements is not otherwise available. In fact most of the code from the slave thread that is now also run when executing BINLOG statements is redundant, or does the wrong thing (like updating binlog position). Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is it to be able to simulate the effect of the slave sql thread? I think the former is the most important one. LOW-LEVEL DESIGN: I think the fix for point 2 is to replace the call to apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct call to ev->apply_event(). However, I need to check two things to be sure this is correct: 1. The existing code does if (!ev->when) ev->when= my_time(0) Need to understand if this is needed/correct or not. 2. The existing code does ev->update_pos(). I think this is redundant for BINLOG statement (as it uses a fake rli structure), but I need to check to make sure. Once this is done, point 1 may no longer be needed. The user can use --base64-output=never when applying the output to a 5.0 server, and omit that option when applying the output to a 5.1 server. There should be no need to omit the format description event in the output when other BINLOG statements for row events are present, as it no longer changes the server id. In fact the format description event is required to be able to execute other BINLOG statements, as its purpose is to define the binary format of the events contained in these statements. Alternatively, we could implement that if the format description event of the source binlog has server version < 5.1, and --base64-output=auto (default), then the format description event is omitted (and should any BINLOG statement be needed, unlikely as it is from a 5.0 server, we will need to throw an error). The binlog format version for 5.0 and 5.1 is actually the same, hence the need to look at server version to guess if the format description event can be omitted. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Progress (by Knielsen): mysqlbinlog: remove undesired effects of format description binlog statement (50)
by worklog-noreply＠askmonty.org 09 Sep '09

09 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: mysqlbinlog: remove undesired effects of format description binlog statement CREATION DATE..: Mon, 17 Aug 2009, 14:12 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 50 (http://askmonty.org/worklog/?tid=50) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 10 ESTIMATE.......: 10 (hours remain) ORIG. ESTIMATE.: 10 PROGRESS NOTES: -=-=(Knielsen - Wed, 09 Sep 2009, 18:31)=-=- Discussed with MySQL developers on Bug#46640. Committed a suggested fix: https://lists.launchpad.net/maria-developers/msg00926.html Worked 8 hours and estimate 10 hours remain (original estimate increased by 18 hours). -=-=(Guest - Wed, 09 Sep 2009, 11:50)=-=- High Level Description modified. --- /tmp/wklog.50.old.11584 2009-09-09 11:50:12.000000000 +0300 +++ /tmp/wklog.50.new.11584 2009-09-09 11:50:12.000000000 +0300 @@ -5,7 +5,11 @@ This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. -2. When one applies "format description binlog statement" at the slave, it -will change the slave's server_id when applied. +2. When one executes a BINLOG statement (containing "format description binlog +statement" or other events), this will cause subsequent statements run in +the same session to be binlogged with the server id of the last event +executed, not with the real server id. This can cause problems like infinite +recursive application in a circular replication topology. This WL is to fix these issues. + -=-=(Knielsen - Tue, 08 Sep 2009, 15:07)=-=- Low Level Design modified. --- /tmp/wklog.50.old.19208 2009-09-08 15:07:09.000000000 +0300 +++ /tmp/wklog.50.new.19208 2009-09-08 15:07:09.000000000 +0300 @@ -1 +1,34 @@ +I think the fix for point 2 is to replace the call to +apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct +call to ev->apply_event(). However, I need to check two things to be sure this +is correct: + +1. The existing code does + + if (!ev->when) ev->when= my_time(0) + +Need to understand if this is needed/correct or not. + +2. The existing code does ev->update_pos(). I think this is redundant for +BINLOG statement (as it uses a fake rli structure), but I need to check to +make sure. + +Once this is done, point 1 may no longer be needed. The user can use +--base64-output=never when applying the output to a 5.0 server, and omit that +option when applying the output to a 5.1 server. There should be no need to +omit the format description event in the output when other BINLOG statements +for row events are present, as it no longer changes the server id. In fact the +format description event is required to be able to execute other BINLOG +statements, as its purpose is to define the binary format of the events +contained in these statements. + +Alternatively, we could implement that if the format description event of the +source binlog has server version < 5.1, and --base64-output=auto (default), +then the format description event is omitted (and should any BINLOG statement +be needed, unlikely as it is from a 5.0 server, we will need to throw an +error). + +The binlog format version for 5.0 and 5.1 is actually the same, hence the need +to look at server version to guess if the format description event can be +omitted. -=-=(Knielsen - Tue, 08 Sep 2009, 14:29)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.17531 2009-09-08 14:29:30.000000000 +0300 +++ /tmp/wklog.50.new.17531 2009-09-08 14:29:30.000000000 +0300 @@ -12,3 +12,37 @@ One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? +First, the problem is not the format description log event. In fact all log +events have their own value of server_id, and every event in the BINLOG +statement is executed with its own id as server_id. + +It seems it was introduced deliberately in the patch for Bug#32407. This patch +makes sql thread and BINLOG statement share code path for executing the binary +event, even though the bug is really about something different (outputting +format description event to allow proper execution of other BINLOG +statements). + +Nevertheless, I think using the server_id in the events of a BINLOG statement +is wrong: + +1. It is different behaviour from executing normal statement-based events not +written using BINLOG. + +2. It causes the possibility for infinite cycle of replication in a cyclic +replication setup, if the server ids in BINLOG are different from all other +server ids in the cycle. + +3. The functionality of applying events with original server id from the event +is still available by pointing the slave thread to the binlog. The +functionality of applying row-based binlog events with server id of the server +executing the BINLOG statements is not otherwise available. + +In fact most of the code from the slave thread that is now also run when +executing BINLOG statements is redundant, or does the wrong thing (like +updating binlog position). + +Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be +able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is +it to be able to simulate the effect of the slave sql thread? I think the +former is the most important one. + -=-=(Knielsen - Mon, 07 Sep 2009, 10:06)=-=- Research worklog, bugs, and existing server code. Worked 2 hours and estimate 0 hours remain (original estimate increased by 2 hours). -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- Dependency created: 39 now depends on 50 -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.11389 2009-08-17 14:13:05.000000000 +0300 +++ /tmp/wklog.50.new.11389 2009-08-17 14:13:05.000000000 +0300 @@ -1 +1,14 @@ +First item +---------- +AFAIU what needs to be done is: +1. record a source server version (it is in the first binlog event). +2. don't emit a BINLOG statement if the recorded version number is 5.0.x or + below. + +and we'll get a 5.0-applicable binlog. + +Second item +----------- +One question that one needs to sort out before disabling server_id change is +why it was put there in the first place? Should it be always removed? DESCRIPTION: According to complaints in BUG#46640: 1. mysqlbinlog will emit a 5.0-incompatible "format description binlog statement" even when reading a binary log that was produced by 5.0. This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. 2. When one executes a BINLOG statement (containing "format description binlog statement" or other events), this will cause subsequent statements run in the same session to be binlogged with the server id of the last event executed, not with the real server id. This can cause problems like infinite recursive application in a circular replication topology. This WL is to fix these issues. HIGH-LEVEL SPECIFICATION: First item ---------- AFAIU what needs to be done is: 1. record a source server version (it is in the first binlog event). 2. don't emit a BINLOG statement if the recorded version number is 5.0.x or below. and we'll get a 5.0-applicable binlog. Second item ----------- One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? First, the problem is not the format description log event. In fact all log events have their own value of server_id, and every event in the BINLOG statement is executed with its own id as server_id. It seems it was introduced deliberately in the patch for Bug#32407. This patch makes sql thread and BINLOG statement share code path for executing the binary event, even though the bug is really about something different (outputting format description event to allow proper execution of other BINLOG statements). Nevertheless, I think using the server_id in the events of a BINLOG statement is wrong: 1. It is different behaviour from executing normal statement-based events not written using BINLOG. 2. It causes the possibility for infinite cycle of replication in a cyclic replication setup, if the server ids in BINLOG are different from all other server ids in the cycle. 3. The functionality of applying events with original server id from the event is still available by pointing the slave thread to the binlog. The functionality of applying row-based binlog events with server id of the server executing the BINLOG statements is not otherwise available. In fact most of the code from the slave thread that is now also run when executing BINLOG statements is redundant, or does the wrong thing (like updating binlog position). Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is it to be able to simulate the effect of the slave sql thread? I think the former is the most important one. LOW-LEVEL DESIGN: I think the fix for point 2 is to replace the call to apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct call to ev->apply_event(). However, I need to check two things to be sure this is correct: 1. The existing code does if (!ev->when) ev->when= my_time(0) Need to understand if this is needed/correct or not. 2. The existing code does ev->update_pos(). I think this is redundant for BINLOG statement (as it uses a fake rli structure), but I need to check to make sure. Once this is done, point 1 may no longer be needed. The user can use --base64-output=never when applying the output to a 5.0 server, and omit that option when applying the output to a 5.1 server. There should be no need to omit the format description event in the output when other BINLOG statements for row events are present, as it no longer changes the server id. In fact the format description event is required to be able to execute other BINLOG statements, as its purpose is to define the binary format of the events contained in these statements. Alternatively, we could implement that if the format description event of the source binlog has server version < 5.1, and --base64-output=auto (default), then the format description event is omitted (and should any BINLOG statement be needed, unlikely as it is from a 5.0 server, we will need to throw an error). The binlog format version for 5.0 and 5.1 is actually the same, hence the need to look at server version to guess if the format description event can be omitted. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2731) Bug#46640
by knielsen＠knielsen-hq.org 09 Sep '09

09 Sep '09

#At lp:maria 2731 knielsen(a)knielsen-hq.org 2009-09-09 Bug#46640: output from mysqlbinlog command in 5.1 breaks replication The BINLOG statement was sharing too much code with the slave SQL thread, introduced with the patch for Bug#32407. This caused statements to be logged with the wrong server_id, the id stored inside the events of the BINLOG statement rather than the id of the running server. Fix by rearranging code a bit so that only relevant parts of the code are executed by the BINLOG statement. modified: mysql-test/extra/binlog_tests/binlog.test mysql-test/suite/binlog/r/binlog_row_binlog.result mysql-test/suite/binlog/r/binlog_stm_binlog.result sql/log_event.cc sql/slave.cc sql/slave.h sql/sql_binlog.cc per-file messages: mysql-test/extra/binlog_tests/binlog.test Add test case. mysql-test/suite/binlog/r/binlog_row_binlog.result Add test case. mysql-test/suite/binlog/r/binlog_stm_binlog.result Add test case. sql/log_event.cc Move rows_event_stmt_clean() call from update_pos() to apply_event(). This in any case makes more sense, and is needed as update_pos() is no longer called when executing BINLOG statements. sql/slave.cc The skip flag is no longer needed, as the code path for BINLOG statement has been cleaned up. sql/slave.h The skip flag is no longer needed, as the code path for BINLOG statement has been cleaned up. sql/sql_binlog.cc Don't invoke the update_pos() code path for the BINLOG statement, as it contains code that is redundant and/or harmful (especially setting thd->server_id). === modified file 'mysql-test/extra/binlog_tests/binlog.test' --- a/mysql-test/extra/binlog_tests/binlog.test 2009-03-25 19:41:16 +0000 +++ b/mysql-test/extra/binlog_tests/binlog.test 2009-09-09 15:27:23 +0000 @@ -258,3 +258,39 @@ dec $it; } show master status /* must show new binlog index after rotating */; drop table t3; + + +# +# Bug#46640: output from mysqlbinlog command in 5.1 breaks replication +# + +connect (fresh,localhost,root,,test); +connection fresh; + +RESET MASTER; +CREATE TABLE t1 (a INT PRIMARY KEY); + +# Format description event, with server_id = 10; +BINLOG ' +3u9kSA8KAAAAZgAAAGoAAAABAAQANS4xLjM1LW1hcmlhLWJldGExLWRlYnVnLWxvZwAAAAAAAAAA +AAAAAAAAAAAAAAAAAADe72RIEzgNAAgAEgAEBAQEEgAAUwAEGggAAAAICAgC +'; + +# What server_id is logged for a statement? Should be our own, not the +# one from the format description event. +INSERT INTO t1 VALUES (1); + +# INSERT INTO t1 VALUES (2), with server_id=20. Check that this is logged +# with our own server id, not the 20 from the BINLOG statement. +BINLOG ' +3u9kSBMUAAAAKQAAAJEBAAAAABoAAAAAAAAABHRlc3QAAnQxAAEDAAA= +3u9kSBcUAAAAIgAAALMBAAAQABoAAAAAAAEAAf/+AgAAAA== +'; + +# Show binlog events to check that server ids are correct. +--replace_column 1 # 2 # 5 # +--replace_regex /Server ver: .*, Binlog ver: .*/Server ver: #, Binlog ver: #/ /table_id: [0-9]+/table_id: #/ +SHOW BINLOG EVENTS; + +DROP TABLE t1; +disconnect fresh; === modified file 'mysql-test/suite/binlog/r/binlog_row_binlog.result' --- a/mysql-test/suite/binlog/r/binlog_row_binlog.result 2009-03-25 19:41:16 +0000 +++ b/mysql-test/suite/binlog/r/binlog_row_binlog.result 2009-09-09 15:27:23 +0000 @@ -1298,3 +1298,27 @@ show master status /* must show new binl File Position Binlog_Do_DB Binlog_Ignore_DB master-bin.000002 106 drop table t3; +RESET MASTER; +CREATE TABLE t1 (a INT PRIMARY KEY); +BINLOG ' +3u9kSA8KAAAAZgAAAGoAAAABAAQANS4xLjM1LW1hcmlhLWJldGExLWRlYnVnLWxvZwAAAAAAAAAA +AAAAAAAAAAAAAAAAAADe72RIEzgNAAgAEgAEBAQEEgAAUwAEGggAAAAICAgC +'; +INSERT INTO t1 VALUES (1); +BINLOG ' +3u9kSBMUAAAAKQAAAJEBAAAAABoAAAAAAAAABHRlc3QAAnQxAAEDAAA= +3u9kSBcUAAAAIgAAALMBAAAQABoAAAAAAAEAAf/+AgAAAA== +'; +SHOW BINLOG EVENTS; +Log_name Pos Event_type Server_id End_log_pos Info +# # Format_desc 1 # Server ver: #, Binlog ver: # +# # Query 1 # use `test`; CREATE TABLE t1 (a INT PRIMARY KEY) +# # Query 1 # use `test`; BEGIN +# # Table_map 1 # table_id: # (test.t1) +# # Write_rows 1 # table_id: # flags: STMT_END_F +# # Query 1 # use `test`; COMMIT +# # Query 1 # use `test`; BEGIN +# # Table_map 1 # table_id: # (test.t1) +# # Write_rows 1 # table_id: # flags: STMT_END_F +# # Query 1 # use `test`; COMMIT +DROP TABLE t1; === modified file 'mysql-test/suite/binlog/r/binlog_stm_binlog.result' --- a/mysql-test/suite/binlog/r/binlog_stm_binlog.result 2009-03-25 19:41:16 +0000 +++ b/mysql-test/suite/binlog/r/binlog_stm_binlog.result 2009-09-09 15:27:23 +0000 @@ -773,3 +773,24 @@ show master status /* must show new binl File Position Binlog_Do_DB Binlog_Ignore_DB master-bin.000002 106 drop table t3; +RESET MASTER; +CREATE TABLE t1 (a INT PRIMARY KEY); +BINLOG ' +3u9kSA8KAAAAZgAAAGoAAAABAAQANS4xLjM1LW1hcmlhLWJldGExLWRlYnVnLWxvZwAAAAAAAAAA +AAAAAAAAAAAAAAAAAADe72RIEzgNAAgAEgAEBAQEEgAAUwAEGggAAAAICAgC +'; +INSERT INTO t1 VALUES (1); +BINLOG ' +3u9kSBMUAAAAKQAAAJEBAAAAABoAAAAAAAAABHRlc3QAAnQxAAEDAAA= +3u9kSBcUAAAAIgAAALMBAAAQABoAAAAAAAEAAf/+AgAAAA== +'; +SHOW BINLOG EVENTS; +Log_name Pos Event_type Server_id End_log_pos Info +# # Format_desc 1 # Server ver: #, Binlog ver: # +# # Query 1 # use `test`; CREATE TABLE t1 (a INT PRIMARY KEY) +# # Query 1 # use `test`; INSERT INTO t1 VALUES (1) +# # Query 1 # use `test`; BEGIN +# # Table_map 1 # table_id: # (test.t1) +# # Write_rows 1 # table_id: # flags: STMT_END_F +# # Query 1 # use `test`; COMMIT +DROP TABLE t1; === modified file 'sql/log_event.cc' --- a/sql/log_event.cc 2009-06-11 17:49:51 +0000 +++ b/sql/log_event.cc 2009-09-09 15:27:23 +0000 @@ -7427,6 +7427,13 @@ int Rows_log_event::do_apply_event(Relay const_cast<Relay_log_info*>(rli)->last_event_start_time= my_time(0); } + if (get_flags(STMT_END_F)) + { + error= rows_event_stmt_cleanup(rli, thd); + if (error) + DBUG_RETURN(error); + } + DBUG_RETURN(0); } @@ -7525,33 +7532,22 @@ Rows_log_event::do_update_pos(Relay_log_ if (get_flags(STMT_END_F)) { - if ((error= rows_event_stmt_cleanup(rli, thd)) == 0) - { - /* - Indicate that a statement is finished. - Step the group log position if we are not in a transaction, - otherwise increase the event log position. - */ - rli->stmt_done(log_pos, when); + /* + Indicate that a statement is finished. + Step the group log position if we are not in a transaction, + otherwise increase the event log position. + */ + rli->stmt_done(log_pos, when); - /* - Clear any errors pushed in thd->net.last_err* if for example "no key - found" (as this is allowed). This is a safety measure; apparently - those errors (e.g. when executing a Delete_rows_log_event of a - non-existing row, like in rpl_row_mystery22.test, - thd->net.last_error = "Can't find record in 't1'" and last_errno=1032) - do not become visible. We still prefer to wipe them out. - */ - thd->clear_error(); - } - else - { - rli->report(ERROR_LEVEL, error, - "Error in %s event: commit of row events failed, " - "table `%s`.`%s`", - get_type_str(), m_table->s->db.str, - m_table->s->table_name.str); - } + /* + Clear any errors pushed in thd->net.last_err* if for example "no key + found" (as this is allowed). This is a safety measure; apparently + those errors (e.g. when executing a Delete_rows_log_event of a + non-existing row, like in rpl_row_mystery22.test, + thd->net.last_error = "Can't find record in 't1'" and last_errno=1032) + do not become visible. We still prefer to wipe them out. + */ + thd->clear_error(); } else { === modified file 'sql/slave.cc' --- a/sql/slave.cc 2009-09-03 14:05:38 +0000 +++ b/sql/slave.cc 2009-09-09 15:27:23 +0000 @@ -1882,8 +1882,7 @@ static int has_temporary_error(THD *thd) @retval 2 No error calling ev->apply_event(), but error calling ev->update_pos(). */ -int apply_event_and_update_pos(Log_event* ev, THD* thd, Relay_log_info* rli, - bool skip) +int apply_event_and_update_pos(Log_event* ev, THD* thd, Relay_log_info* rli) { int exec_res= 0; @@ -1928,37 +1927,32 @@ int apply_event_and_update_pos(Log_event ev->when= my_time(0); ev->thd = thd; // because up to this point, ev->thd == 0 - if (skip) - { - int reason= ev->shall_skip(rli); - if (reason == Log_event::EVENT_SKIP_COUNT) - --rli->slave_skip_counter; - pthread_mutex_unlock(&rli->data_lock); - if (reason == Log_event::EVENT_SKIP_NOT) - exec_res= ev->apply_event(rli); + int reason= ev->shall_skip(rli); + if (reason == Log_event::EVENT_SKIP_COUNT) + --rli->slave_skip_counter; + pthread_mutex_unlock(&rli->data_lock); + if (reason == Log_event::EVENT_SKIP_NOT) + exec_res= ev->apply_event(rli); #ifndef DBUG_OFF - /* - This only prints information to the debug trace. + /* + This only prints information to the debug trace. - TODO: Print an informational message to the error log? - */ - static const char *const explain[] = { - // EVENT_SKIP_NOT, - "not skipped", - // EVENT_SKIP_IGNORE, - "skipped because event should be ignored", - // EVENT_SKIP_COUNT - "skipped because event skip counter was non-zero" - }; - DBUG_PRINT("info", ("OPTION_BEGIN: %d; IN_STMT: %d", - thd->options & OPTION_BEGIN ? 1 : 0, - rli->get_flag(Relay_log_info::IN_STMT))); - DBUG_PRINT("skip_event", ("%s event was %s", - ev->get_type_str(), explain[reason])); + TODO: Print an informational message to the error log? + */ + static const char *const explain[] = { + // EVENT_SKIP_NOT, + "not skipped", + // EVENT_SKIP_IGNORE, + "skipped because event should be ignored", + // EVENT_SKIP_COUNT + "skipped because event skip counter was non-zero" + }; + DBUG_PRINT("info", ("OPTION_BEGIN: %d; IN_STMT: %d", + thd->options & OPTION_BEGIN ? 1 : 0, + rli->get_flag(Relay_log_info::IN_STMT))); + DBUG_PRINT("skip_event", ("%s event was %s", + ev->get_type_str(), explain[reason])); #endif - } - else - exec_res= ev->apply_event(rli); DBUG_PRINT("info", ("apply_event error = %d", exec_res)); if (exec_res == 0) @@ -2078,7 +2072,7 @@ static int exec_relay_log_event(THD* thd delete ev; DBUG_RETURN(1); } - exec_res= apply_event_and_update_pos(ev, thd, rli, TRUE); + exec_res= apply_event_and_update_pos(ev, thd, rli); /* Format_description_log_event should not be deleted because it will be === modified file 'sql/slave.h' --- a/sql/slave.h 2008-12-10 09:02:25 +0000 +++ b/sql/slave.h 2009-09-09 15:27:23 +0000 @@ -190,8 +190,7 @@ int purge_relay_logs(Relay_log_info* rli void set_slave_thread_options(THD* thd); void set_slave_thread_default_charset(THD *thd, Relay_log_info const *rli); void rotate_relay_log(Master_info* mi); -int apply_event_and_update_pos(Log_event* ev, THD* thd, Relay_log_info* rli, - bool skip); +int apply_event_and_update_pos(Log_event* ev, THD* thd, Relay_log_info* rli); pthread_handler_t handle_slave_io(void *arg); pthread_handler_t handle_slave_sql(void *arg); === modified file 'sql/sql_binlog.cc' --- a/sql/sql_binlog.cc 2009-05-06 12:03:24 +0000 +++ b/sql/sql_binlog.cc 2009-09-09 15:27:23 +0000 @@ -56,17 +56,21 @@ void mysql_client_binlog_statement(THD* Format_description_event. */ my_bool have_fd_event= TRUE; - if (!thd->rli_fake) + int err; + Relay_log_info *rli; + + rli= thd->rli_fake; + if (!rli) { - thd->rli_fake= new Relay_log_info; + rli= thd->rli_fake= new Relay_log_info; #ifdef HAVE_valgrind - thd->rli_fake->is_fake= TRUE; + rli->is_fake= TRUE; #endif have_fd_event= FALSE; } - if (thd->rli_fake && !thd->rli_fake->relay_log.description_event_for_exec) + if (rli && !rli->relay_log.description_event_for_exec) { - thd->rli_fake->relay_log.description_event_for_exec= + rli->relay_log.description_event_for_exec= new Format_description_log_event(4); have_fd_event= FALSE; } @@ -78,16 +82,16 @@ void mysql_client_binlog_statement(THD* /* Out of memory check */ - if (!(thd->rli_fake && - thd->rli_fake->relay_log.description_event_for_exec && + if (!(rli && + rli->relay_log.description_event_for_exec && buf)) { my_error(ER_OUTOFMEMORY, MYF(0), 1); /* needed 1 bytes */ goto end; } - thd->rli_fake->sql_thd= thd; - thd->rli_fake->no_storage= TRUE; + rli->sql_thd= thd; + rli->no_storage= TRUE; for (char const *strptr= thd->lex->comment.str ; strptr < thd->lex->comment.str + thd->lex->comment.length ; ) @@ -170,8 +174,7 @@ void mysql_client_binlog_statement(THD* } ev= Log_event::read_log_event(bufptr, event_len, &error, - thd->rli_fake->relay_log. - description_event_for_exec); + rli->relay_log.description_event_for_exec); DBUG_PRINT("info",("binlog base64 err=%s", error)); if (!ev) @@ -209,9 +212,27 @@ void mysql_client_binlog_statement(THD* reporting. */ #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) - if (apply_event_and_update_pos(ev, thd, thd->rli_fake, FALSE)) + err= ev->apply_event(rli); +#else + err= 0; +#endif + if (ev->get_type_code() == FORMAT_DESCRIPTION_EVENT) + { + /* + Install this as the format description event to use for any + following events. + */ + delete rli->relay_log.description_event_for_exec; + rli->relay_log.description_event_for_exec= + static_cast<Format_description_log_event *>(ev); + } + else { delete ev; + } + ev= 0; + if (err) + { /* TODO: Maybe a better error message since the BINLOG statement now contains several events. @@ -219,17 +240,6 @@ void mysql_client_binlog_statement(THD* my_error(ER_UNKNOWN_ERROR, MYF(0), "Error executing BINLOG statement"); goto end; } -#endif - - /* - Format_description_log_event should not be deleted because it - will be used to read info about the relay log's format; it - will be deleted when the SQL thread does not need it, - i.e. when this thread terminates. - */ - if (ev->get_type_code() != FORMAT_DESCRIPTION_EVENT) - delete ev; - ev= 0; } } @@ -238,7 +248,7 @@ void mysql_client_binlog_statement(THD* my_ok(thd); end: - thd->rli_fake->clear_tables_to_lock(); + rli->clear_tables_to_lock(); my_free(buf, MYF(MY_ALLOW_ZERO_PTR)); DBUG_VOID_RETURN; }

1 0

[Maria-developers] Updated (by Guest): mysqlbinlog: remove undesired effects of format description binlog statement (50)
by worklog-noreply＠askmonty.org 09 Sep '09

09 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: mysqlbinlog: remove undesired effects of format description binlog statement CREATION DATE..: Mon, 17 Aug 2009, 14:12 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 50 (http://askmonty.org/worklog/?tid=50) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 2 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Wed, 09 Sep 2009, 11:50)=-=- High Level Description modified. --- /tmp/wklog.50.old.11584 2009-09-09 11:50:12.000000000 +0300 +++ /tmp/wklog.50.new.11584 2009-09-09 11:50:12.000000000 +0300 @@ -5,7 +5,11 @@ This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. -2. When one applies "format description binlog statement" at the slave, it -will change the slave's server_id when applied. +2. When one executes a BINLOG statement (containing "format description binlog +statement" or other events), this will cause subsequent statements run in +the same session to be binlogged with the server id of the last event +executed, not with the real server id. This can cause problems like infinite +recursive application in a circular replication topology. This WL is to fix these issues. + -=-=(Knielsen - Tue, 08 Sep 2009, 15:07)=-=- Low Level Design modified. --- /tmp/wklog.50.old.19208 2009-09-08 15:07:09.000000000 +0300 +++ /tmp/wklog.50.new.19208 2009-09-08 15:07:09.000000000 +0300 @@ -1 +1,34 @@ +I think the fix for point 2 is to replace the call to +apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct +call to ev->apply_event(). However, I need to check two things to be sure this +is correct: + +1. The existing code does + + if (!ev->when) ev->when= my_time(0) + +Need to understand if this is needed/correct or not. + +2. The existing code does ev->update_pos(). I think this is redundant for +BINLOG statement (as it uses a fake rli structure), but I need to check to +make sure. + +Once this is done, point 1 may no longer be needed. The user can use +--base64-output=never when applying the output to a 5.0 server, and omit that +option when applying the output to a 5.1 server. There should be no need to +omit the format description event in the output when other BINLOG statements +for row events are present, as it no longer changes the server id. In fact the +format description event is required to be able to execute other BINLOG +statements, as its purpose is to define the binary format of the events +contained in these statements. + +Alternatively, we could implement that if the format description event of the +source binlog has server version < 5.1, and --base64-output=auto (default), +then the format description event is omitted (and should any BINLOG statement +be needed, unlikely as it is from a 5.0 server, we will need to throw an +error). + +The binlog format version for 5.0 and 5.1 is actually the same, hence the need +to look at server version to guess if the format description event can be +omitted. -=-=(Knielsen - Tue, 08 Sep 2009, 14:29)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.17531 2009-09-08 14:29:30.000000000 +0300 +++ /tmp/wklog.50.new.17531 2009-09-08 14:29:30.000000000 +0300 @@ -12,3 +12,37 @@ One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? +First, the problem is not the format description log event. In fact all log +events have their own value of server_id, and every event in the BINLOG +statement is executed with its own id as server_id. + +It seems it was introduced deliberately in the patch for Bug#32407. This patch +makes sql thread and BINLOG statement share code path for executing the binary +event, even though the bug is really about something different (outputting +format description event to allow proper execution of other BINLOG +statements). + +Nevertheless, I think using the server_id in the events of a BINLOG statement +is wrong: + +1. It is different behaviour from executing normal statement-based events not +written using BINLOG. + +2. It causes the possibility for infinite cycle of replication in a cyclic +replication setup, if the server ids in BINLOG are different from all other +server ids in the cycle. + +3. The functionality of applying events with original server id from the event +is still available by pointing the slave thread to the binlog. The +functionality of applying row-based binlog events with server id of the server +executing the BINLOG statements is not otherwise available. + +In fact most of the code from the slave thread that is now also run when +executing BINLOG statements is redundant, or does the wrong thing (like +updating binlog position). + +Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be +able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is +it to be able to simulate the effect of the slave sql thread? I think the +former is the most important one. + -=-=(Knielsen - Mon, 07 Sep 2009, 10:06)=-=- Research worklog, bugs, and existing server code. Worked 2 hours and estimate 0 hours remain (original estimate increased by 2 hours). -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- Dependency created: 39 now depends on 50 -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.11389 2009-08-17 14:13:05.000000000 +0300 +++ /tmp/wklog.50.new.11389 2009-08-17 14:13:05.000000000 +0300 @@ -1 +1,14 @@ +First item +---------- +AFAIU what needs to be done is: +1. record a source server version (it is in the first binlog event). +2. don't emit a BINLOG statement if the recorded version number is 5.0.x or + below. + +and we'll get a 5.0-applicable binlog. + +Second item +----------- +One question that one needs to sort out before disabling server_id change is +why it was put there in the first place? Should it be always removed? DESCRIPTION: According to complaints in BUG#46640: 1. mysqlbinlog will emit a 5.0-incompatible "format description binlog statement" even when reading a binary log that was produced by 5.0. This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. 2. When one executes a BINLOG statement (containing "format description binlog statement" or other events), this will cause subsequent statements run in the same session to be binlogged with the server id of the last event executed, not with the real server id. This can cause problems like infinite recursive application in a circular replication topology. This WL is to fix these issues. HIGH-LEVEL SPECIFICATION: First item ---------- AFAIU what needs to be done is: 1. record a source server version (it is in the first binlog event). 2. don't emit a BINLOG statement if the recorded version number is 5.0.x or below. and we'll get a 5.0-applicable binlog. Second item ----------- One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? First, the problem is not the format description log event. In fact all log events have their own value of server_id, and every event in the BINLOG statement is executed with its own id as server_id. It seems it was introduced deliberately in the patch for Bug#32407. This patch makes sql thread and BINLOG statement share code path for executing the binary event, even though the bug is really about something different (outputting format description event to allow proper execution of other BINLOG statements). Nevertheless, I think using the server_id in the events of a BINLOG statement is wrong: 1. It is different behaviour from executing normal statement-based events not written using BINLOG. 2. It causes the possibility for infinite cycle of replication in a cyclic replication setup, if the server ids in BINLOG are different from all other server ids in the cycle. 3. The functionality of applying events with original server id from the event is still available by pointing the slave thread to the binlog. The functionality of applying row-based binlog events with server id of the server executing the BINLOG statements is not otherwise available. In fact most of the code from the slave thread that is now also run when executing BINLOG statements is redundant, or does the wrong thing (like updating binlog position). Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is it to be able to simulate the effect of the slave sql thread? I think the former is the most important one. LOW-LEVEL DESIGN: I think the fix for point 2 is to replace the call to apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct call to ev->apply_event(). However, I need to check two things to be sure this is correct: 1. The existing code does if (!ev->when) ev->when= my_time(0) Need to understand if this is needed/correct or not. 2. The existing code does ev->update_pos(). I think this is redundant for BINLOG statement (as it uses a fake rli structure), but I need to check to make sure. Once this is done, point 1 may no longer be needed. The user can use --base64-output=never when applying the output to a 5.0 server, and omit that option when applying the output to a 5.1 server. There should be no need to omit the format description event in the output when other BINLOG statements for row events are present, as it no longer changes the server id. In fact the format description event is required to be able to execute other BINLOG statements, as its purpose is to define the binary format of the events contained in these statements. Alternatively, we could implement that if the format description event of the source binlog has server version < 5.1, and --base64-output=auto (default), then the format description event is omitted (and should any BINLOG statement be needed, unlikely as it is from a 5.0 server, we will need to throw an error). The binlog format version for 5.0 and 5.1 is actually the same, hence the need to look at server version to guess if the format description event can be omitted. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): mysqlbinlog: remove undesired effects of format description binlog statement (50)
by worklog-noreply＠askmonty.org 09 Sep '09

09 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: mysqlbinlog: remove undesired effects of format description binlog statement CREATION DATE..: Mon, 17 Aug 2009, 14:12 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 50 (http://askmonty.org/worklog/?tid=50) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 2 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Wed, 09 Sep 2009, 11:50)=-=- High Level Description modified. --- /tmp/wklog.50.old.11584 2009-09-09 11:50:12.000000000 +0300 +++ /tmp/wklog.50.new.11584 2009-09-09 11:50:12.000000000 +0300 @@ -5,7 +5,11 @@ This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. -2. When one applies "format description binlog statement" at the slave, it -will change the slave's server_id when applied. +2. When one executes a BINLOG statement (containing "format description binlog +statement" or other events), this will cause subsequent statements run in +the same session to be binlogged with the server id of the last event +executed, not with the real server id. This can cause problems like infinite +recursive application in a circular replication topology. This WL is to fix these issues. + -=-=(Knielsen - Tue, 08 Sep 2009, 15:07)=-=- Low Level Design modified. --- /tmp/wklog.50.old.19208 2009-09-08 15:07:09.000000000 +0300 +++ /tmp/wklog.50.new.19208 2009-09-08 15:07:09.000000000 +0300 @@ -1 +1,34 @@ +I think the fix for point 2 is to replace the call to +apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct +call to ev->apply_event(). However, I need to check two things to be sure this +is correct: + +1. The existing code does + + if (!ev->when) ev->when= my_time(0) + +Need to understand if this is needed/correct or not. + +2. The existing code does ev->update_pos(). I think this is redundant for +BINLOG statement (as it uses a fake rli structure), but I need to check to +make sure. + +Once this is done, point 1 may no longer be needed. The user can use +--base64-output=never when applying the output to a 5.0 server, and omit that +option when applying the output to a 5.1 server. There should be no need to +omit the format description event in the output when other BINLOG statements +for row events are present, as it no longer changes the server id. In fact the +format description event is required to be able to execute other BINLOG +statements, as its purpose is to define the binary format of the events +contained in these statements. + +Alternatively, we could implement that if the format description event of the +source binlog has server version < 5.1, and --base64-output=auto (default), +then the format description event is omitted (and should any BINLOG statement +be needed, unlikely as it is from a 5.0 server, we will need to throw an +error). + +The binlog format version for 5.0 and 5.1 is actually the same, hence the need +to look at server version to guess if the format description event can be +omitted. -=-=(Knielsen - Tue, 08 Sep 2009, 14:29)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.17531 2009-09-08 14:29:30.000000000 +0300 +++ /tmp/wklog.50.new.17531 2009-09-08 14:29:30.000000000 +0300 @@ -12,3 +12,37 @@ One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? +First, the problem is not the format description log event. In fact all log +events have their own value of server_id, and every event in the BINLOG +statement is executed with its own id as server_id. + +It seems it was introduced deliberately in the patch for Bug#32407. This patch +makes sql thread and BINLOG statement share code path for executing the binary +event, even though the bug is really about something different (outputting +format description event to allow proper execution of other BINLOG +statements). + +Nevertheless, I think using the server_id in the events of a BINLOG statement +is wrong: + +1. It is different behaviour from executing normal statement-based events not +written using BINLOG. + +2. It causes the possibility for infinite cycle of replication in a cyclic +replication setup, if the server ids in BINLOG are different from all other +server ids in the cycle. + +3. The functionality of applying events with original server id from the event +is still available by pointing the slave thread to the binlog. The +functionality of applying row-based binlog events with server id of the server +executing the BINLOG statements is not otherwise available. + +In fact most of the code from the slave thread that is now also run when +executing BINLOG statements is redundant, or does the wrong thing (like +updating binlog position). + +Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be +able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is +it to be able to simulate the effect of the slave sql thread? I think the +former is the most important one. + -=-=(Knielsen - Mon, 07 Sep 2009, 10:06)=-=- Research worklog, bugs, and existing server code. Worked 2 hours and estimate 0 hours remain (original estimate increased by 2 hours). -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- Dependency created: 39 now depends on 50 -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.11389 2009-08-17 14:13:05.000000000 +0300 +++ /tmp/wklog.50.new.11389 2009-08-17 14:13:05.000000000 +0300 @@ -1 +1,14 @@ +First item +---------- +AFAIU what needs to be done is: +1. record a source server version (it is in the first binlog event). +2. don't emit a BINLOG statement if the recorded version number is 5.0.x or + below. + +and we'll get a 5.0-applicable binlog. + +Second item +----------- +One question that one needs to sort out before disabling server_id change is +why it was put there in the first place? Should it be always removed? DESCRIPTION: According to complaints in BUG#46640: 1. mysqlbinlog will emit a 5.0-incompatible "format description binlog statement" even when reading a binary log that was produced by 5.0. This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. 2. When one executes a BINLOG statement (containing "format description binlog statement" or other events), this will cause subsequent statements run in the same session to be binlogged with the server id of the last event executed, not with the real server id. This can cause problems like infinite recursive application in a circular replication topology. This WL is to fix these issues. HIGH-LEVEL SPECIFICATION: First item ---------- AFAIU what needs to be done is: 1. record a source server version (it is in the first binlog event). 2. don't emit a BINLOG statement if the recorded version number is 5.0.x or below. and we'll get a 5.0-applicable binlog. Second item ----------- One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? First, the problem is not the format description log event. In fact all log events have their own value of server_id, and every event in the BINLOG statement is executed with its own id as server_id. It seems it was introduced deliberately in the patch for Bug#32407. This patch makes sql thread and BINLOG statement share code path for executing the binary event, even though the bug is really about something different (outputting format description event to allow proper execution of other BINLOG statements). Nevertheless, I think using the server_id in the events of a BINLOG statement is wrong: 1. It is different behaviour from executing normal statement-based events not written using BINLOG. 2. It causes the possibility for infinite cycle of replication in a cyclic replication setup, if the server ids in BINLOG are different from all other server ids in the cycle. 3. The functionality of applying events with original server id from the event is still available by pointing the slave thread to the binlog. The functionality of applying row-based binlog events with server id of the server executing the BINLOG statements is not otherwise available. In fact most of the code from the slave thread that is now also run when executing BINLOG statements is redundant, or does the wrong thing (like updating binlog position). Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is it to be able to simulate the effect of the slave sql thread? I think the former is the most important one. LOW-LEVEL DESIGN: I think the fix for point 2 is to replace the call to apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct call to ev->apply_event(). However, I need to check two things to be sure this is correct: 1. The existing code does if (!ev->when) ev->when= my_time(0) Need to understand if this is needed/correct or not. 2. The existing code does ev->update_pos(). I think this is redundant for BINLOG statement (as it uses a fake rli structure), but I need to check to make sure. Once this is done, point 1 may no longer be needed. The user can use --base64-output=never when applying the output to a 5.0 server, and omit that option when applying the output to a 5.1 server. There should be no need to omit the format description event in the output when other BINLOG statements for row events are present, as it no longer changes the server id. In fact the format description event is required to be able to execute other BINLOG statements, as its purpose is to define the binary format of the events contained in these statements. Alternatively, we could implement that if the format description event of the source binlog has server version < 5.1, and --base64-output=auto (default), then the format description event is omitted (and should any BINLOG statement be needed, unlikely as it is from a 5.0 server, we will need to throw an error). The binlog format version for 5.0 and 5.1 is actually the same, hence the need to look at server version to guess if the format description event can be omitted. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

Re: [Maria-developers] Rev 2721: Merge xtradb-7 -> MariaDB in file:///home/psergey/bzr-new/mysql-5.1-maria-contd2/
by Sergey Petrunya 08 Sep '09

08 Sep '09

Hi Peter, On Tue, Sep 08, 2009 at 06:33:00PM +0200, Peter Laursen wrote: > @Sergey > . > does this mean that we can now check out code, compile and XtraDB will be > functional? > (cc: to Khusboo as she is assigned as build-engineer for the Windows MariaDB > planned to be provided by Webyog) No, not yet: I haven't pushed this yet, because the result of the merge has certain issues. There will be more commits and then push. Besides the merge, we have also two known windows problems, as mentioned here: http://askmonty.org/wiki/index.php/MariaDB_5.1_TODO#Bugfixes We're working on these also. Do you want me to notify you when lp:maria is ready for windows packaging? BR Sergey -- Sergey Petrunia, Software Developer Monty Program AB, http://askmonty.org Blog: http://s.petrunia.net/blog

2 2