[Commits] 08e32aa6f89: MDEV-19708 RBR replication looses data silently ignoring important column attributes

revision-id: 08e32aa6f8993b7b89c81078a49554ba1cabcf31 (mariadb-10.4.4-171-g08e32aa6f89) parent(s): 984d7100cdab91fb23d97c05e8b6329a90fe1583 author: Sachin committer: Sachin timestamp: 2019-06-25 17:38:28 +0530 message: MDEV-19708 RBR replication looses data silently ignoring important column attributes Cherry-pick the commits the mysql and some changes. WL#4618 RBR: extended table metadata in the binary log This patch extends Table Map Event. It appends some new fields for more metadata. The new metadata includes: - Signedness of Numberic Columns - Character Set of Character Columns and Binary Columns - Column Name - String Value of SET Columns - String Value of ENUM Columns - Primary Key - Geometry Type Some of them are optional, the patch introduces a GLOBAL system variable to control it. It is binlog_row_metadata. - Scope: GLOBAL - Dynamic: Yes - Type: ENUM - Values: {MINIMAL, FULL} - Default: MINIMAL Only Signedness, character set and geometry type are logged if it is MINIMAL. Otherwise all of them are logged. --- client/client_priv.h | 2 + client/mysqlbinlog.cc | 6 + mysql-test/include/grep_pattern.inc | 83 ++++ mysql-test/main/mysqlbinlog_row_compressed.result | 154 ++++---- mysql-test/main/mysqlbinlog_row_minimal.result | 168 ++++----- mysql-test/main/mysqld--help.result | 6 + .../binlog/include/print_optional_metadata.inc | 32 ++ .../r/binlog_table_map_optional_metadata.result | 401 ++++++++++++++++++++ .../t/binlog_table_map_optional_metadata.test | 348 +++++++++++++++++ .../sys_vars/r/binlog_row_metadata_basic.result | 85 +++++ .../sys_vars/t/binlog_row_metadata_basic.test | 124 ++++++ sql/log_event.cc | 335 +++++++++++++++- sql/log_event.h | 327 ++++++++++++++++ sql/log_event_client.cc | 355 ++++++++++++++++- sql/log_event_server.cc | 420 ++++++++++++++++++++- sql/mysqld.cc | 1 + sql/mysqld.h | 1 + sql/sql_class.h | 5 + sql/sys_vars.cc | 12 + 19 files changed, 2698 insertions(+), 167 deletions(-) diff --git a/client/client_priv.h b/client/client_priv.h index 5b6b31ccda9..8b8c217e207 100644 --- a/client/client_priv.h +++ b/client/client_priv.h @@ -101,6 +101,8 @@ enum options_client OPT_SSL_CRL, OPT_SSL_CRLPATH, OPT_PRINT_ROW_COUNT, OPT_PRINT_ROW_EVENT_POSITIONS, OPT_SHUTDOWN_WAIT_FOR_SLAVES, + OPT_PRINT_TABLE_METADATA, + /* Add new option above this */ OPT_MAX_CLIENT_OPTION /* should be always the last */ }; diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index 090d743c61b..759cc2d66e7 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -143,6 +143,7 @@ static const char* dirname_for_local_load= 0; static bool opt_skip_annotate_row_events= 0; static my_bool opt_flashback; +static bool opt_print_table_metadata; #ifdef WHEN_FLASHBACK_REVIEW_READY static my_bool opt_flashback_review; static char *flashback_review_dbname, *flashback_review_tablename; @@ -1094,6 +1095,7 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, print_event_info->hexdump_from= pos; print_event_info->base64_output_mode= opt_base64_output_mode; + print_event_info->print_table_metadata= opt_print_table_metadata; DBUG_PRINT("debug", ("event_type: %s", ev->get_type_str())); @@ -1786,6 +1788,10 @@ Example: rewrite-db='from->to'.", (uchar**) &opt_skip_annotate_row_events, (uchar**) &opt_skip_annotate_row_events, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"print-table-metadata", OPT_PRINT_TABLE_METADATA, + "Print metadata stored in Table_map_log_event", + &opt_print_table_metadata, &opt_print_table_metadata, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0} }; diff --git a/mysql-test/include/grep_pattern.inc b/mysql-test/include/grep_pattern.inc new file mode 100644 index 00000000000..bdbca47fa9f --- /dev/null +++ b/mysql-test/include/grep_pattern.inc @@ -0,0 +1,83 @@ +# ==== Purpose ==== +# +# Grep for a pattern in a file in a portable manner. +# +# WARNING: Use include/assert_grep.inc instead, if at all possible. +# That allows you to assert the property being tested in a much more +# precise way. It also does not depend on the result file, so your +# test becomes more readable/maintainable. It also produces better +# debug output if the test fails. +# +# ==== Usage ==== +# +# --let $grep_pattern= PERL_REGEX +# --let $grep_file= FILENAME +# [--let $grep_output= print_each | print_count | boolean] +# +# Parameters: +# +# $grep_pattern +# The pattern to search for. This can be a perl regex. +# +# $grep_file +# The file to search in. +# +# $grep_output +# The format of the output. Can be one of: +# - print_each: prints each line, and a count. +# - print_count: print just a count. +# - boolean: print only whether something was found or not. +# If this is not specified, print_each is used. + +#--let $include_filename= grep_pattern.inc +#--source include/begin_include_file.inc + + +if ($grep_output) +{ + --let _GP_GREP_OUTPUT= $grep_output +} +if (!$grep_output) +{ + --let _GP_GREP_OUTPUT= print_each +} +--let _GP_GREP_PATTERN= $grep_pattern +--let _GP_GREP_FILE= $grep_file + +--perl + use strict; + my $file= $ENV{'_GP_GREP_FILE'} or die "grep_file is not set"; + my $pattern= $ENV{'_GP_GREP_PATTERN'} or die "grep_pattern is not set"; + open(FILE, "$file") or die("Unable to open $file: $!\n"); + my $count = 0; + my $output = $ENV{'_GP_GREP_OUTPUT'}; + if ($output eq 'print_each') { + print "Matching lines are:\n"; + } + while (<FILE>) { + my $line = $_; + if ($line =~ /$pattern/) { + if ($output eq 'print_each') { + print $line; + } + $count++; + if ($output eq 'boolean') { + last; + } + } + } + if ($count == 0 && $output eq 'print_each') { + print "None\n"; + } + if ($output eq 'boolean') { + print $count ? "Pattern found.\n" : "Pattern not found.\n"; + } + else { + print "Occurrences of '$pattern' in the input file: $count\n"; + } + close(FILE) or die "Error closing $file: $!"; +EOF + + +#--source include/end_include_file.inc + diff --git a/mysql-test/main/mysqlbinlog_row_compressed.result b/mysql-test/main/mysqlbinlog_row_compressed.result index 8cf52f5b826..37b6b2ea7ee 100644 --- a/mysql-test/main/mysqlbinlog_row_compressed.result +++ b/mysql-test/main/mysqlbinlog_row_compressed.result @@ -59,9 +59,9 @@ BEGIN # at 843 #<date> server id 1 end_log_pos 843 CRC32 XXX Annotate_rows: #Q> INSERT INTO t1 VALUES (10, 1, 2, 3, 4, 5, 6, 7, "") -#<date> server id 1 end_log_pos 899 CRC32 XXX Table_map: `test`.`t1` mapped to number num -# at 899 -#<date> server id 1 end_log_pos 967 CRC32 XXX Write_compressed_rows: table id 32 flags: STMT_END_F +#<date> server id 1 end_log_pos 905 CRC32 XXX Table_map: `test`.`t1` mapped to number num +# at 905 +#<date> server id 1 end_log_pos 973 CRC32 XXX Write_compressed_rows: table id 32 flags: STMT_END_F ### INSERT INTO `test`.`t1` ### SET ### @1=10 /* INT meta=0 nullable=0 is_null=0 */ @@ -74,23 +74,23 @@ BEGIN ### @8=7 /* INT meta=0 nullable=1 is_null=0 */ ### @9='' /* STRING(1) meta=65025 nullable=1 is_null=0 */ # Number of rows: 1 -# at 967 -#<date> server id 1 end_log_pos 1040 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 +# at 973 +#<date> server id 1 end_log_pos 1046 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 SET TIMESTAMP=X/*!*/; COMMIT /*!*/; -# at 1040 -#<date> server id 1 end_log_pos 1082 CRC32 XXX GTID 0-1-4 +# at 1046 +#<date> server id 1 end_log_pos 1088 CRC32 XXX GTID 0-1-4 /*!100001 SET @@session.gtid_seq_no=4*//*!*/; BEGIN /*!*/; -# at 1082 -# at 1158 -#<date> server id 1 end_log_pos 1158 CRC32 XXX Annotate_rows: +# at 1088 +# at 1164 +#<date> server id 1 end_log_pos 1164 CRC32 XXX Annotate_rows: #Q> INSERT INTO t1 VALUES (11, 1, 2, 3, 4, 5, 6, 7, NULL) -#<date> server id 1 end_log_pos 1214 CRC32 XXX Table_map: `test`.`t1` mapped to number num -# at 1214 -#<date> server id 1 end_log_pos 1281 CRC32 XXX Write_compressed_rows: table id 32 flags: STMT_END_F +#<date> server id 1 end_log_pos 1226 CRC32 XXX Table_map: `test`.`t1` mapped to number num +# at 1226 +#<date> server id 1 end_log_pos 1293 CRC32 XXX Write_compressed_rows: table id 32 flags: STMT_END_F ### INSERT INTO `test`.`t1` ### SET ### @1=11 /* INT meta=0 nullable=0 is_null=0 */ @@ -103,23 +103,23 @@ BEGIN ### @8=7 /* INT meta=0 nullable=1 is_null=0 */ ### @9=NULL /* STRING(1) meta=65025 nullable=1 is_null=1 */ # Number of rows: 1 -# at 1281 -#<date> server id 1 end_log_pos 1354 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 +# at 1293 +#<date> server id 1 end_log_pos 1366 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 SET TIMESTAMP=X/*!*/; COMMIT /*!*/; -# at 1354 -#<date> server id 1 end_log_pos 1396 CRC32 XXX GTID 0-1-5 +# at 1366 +#<date> server id 1 end_log_pos 1408 CRC32 XXX GTID 0-1-5 /*!100001 SET @@session.gtid_seq_no=5*//*!*/; BEGIN /*!*/; -# at 1396 -# at 1474 -#<date> server id 1 end_log_pos 1474 CRC32 XXX Annotate_rows: +# at 1408 +# at 1486 +#<date> server id 1 end_log_pos 1486 CRC32 XXX Annotate_rows: #Q> INSERT INTO t1 VALUES (12, 1, 2, 3, NULL, 5, 6, 7, "A") -#<date> server id 1 end_log_pos 1530 CRC32 XXX Table_map: `test`.`t1` mapped to number num -# at 1530 -#<date> server id 1 end_log_pos 1596 CRC32 XXX Write_compressed_rows: table id 32 flags: STMT_END_F +#<date> server id 1 end_log_pos 1548 CRC32 XXX Table_map: `test`.`t1` mapped to number num +# at 1548 +#<date> server id 1 end_log_pos 1614 CRC32 XXX Write_compressed_rows: table id 32 flags: STMT_END_F ### INSERT INTO `test`.`t1` ### SET ### @1=12 /* INT meta=0 nullable=0 is_null=0 */ @@ -132,23 +132,23 @@ BEGIN ### @8=7 /* INT meta=0 nullable=1 is_null=0 */ ### @9='A' /* STRING(1) meta=65025 nullable=1 is_null=0 */ # Number of rows: 1 -# at 1596 -#<date> server id 1 end_log_pos 1669 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 +# at 1614 +#<date> server id 1 end_log_pos 1687 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 SET TIMESTAMP=X/*!*/; COMMIT /*!*/; -# at 1669 -#<date> server id 1 end_log_pos 1711 CRC32 XXX GTID 0-1-6 +# at 1687 +#<date> server id 1 end_log_pos 1729 CRC32 XXX GTID 0-1-6 /*!100001 SET @@session.gtid_seq_no=6*//*!*/; BEGIN /*!*/; -# at 1711 -# at 1786 -#<date> server id 1 end_log_pos 1786 CRC32 XXX Annotate_rows: +# at 1729 +# at 1804 +#<date> server id 1 end_log_pos 1804 CRC32 XXX Annotate_rows: #Q> INSERT INTO t1 VALUES (13, 1, 2, 3, 0, 5, 6, 7, "A") -#<date> server id 1 end_log_pos 1842 CRC32 XXX Table_map: `test`.`t1` mapped to number num -# at 1842 -#<date> server id 1 end_log_pos 1909 CRC32 XXX Write_compressed_rows: table id 32 flags: STMT_END_F +#<date> server id 1 end_log_pos 1866 CRC32 XXX Table_map: `test`.`t1` mapped to number num +# at 1866 +#<date> server id 1 end_log_pos 1933 CRC32 XXX Write_compressed_rows: table id 32 flags: STMT_END_F ### INSERT INTO `test`.`t1` ### SET ### @1=13 /* INT meta=0 nullable=0 is_null=0 */ @@ -161,23 +161,23 @@ BEGIN ### @8=7 /* INT meta=0 nullable=1 is_null=0 */ ### @9='A' /* STRING(1) meta=65025 nullable=1 is_null=0 */ # Number of rows: 1 -# at 1909 -#<date> server id 1 end_log_pos 1982 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 +# at 1933 +#<date> server id 1 end_log_pos 2006 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 SET TIMESTAMP=X/*!*/; COMMIT /*!*/; -# at 1982 -#<date> server id 1 end_log_pos 2024 CRC32 XXX GTID 0-1-7 +# at 2006 +#<date> server id 1 end_log_pos 2048 CRC32 XXX GTID 0-1-7 /*!100001 SET @@session.gtid_seq_no=7*//*!*/; BEGIN /*!*/; -# at 2024 -# at 2078 -#<date> server id 1 end_log_pos 2078 CRC32 XXX Annotate_rows: +# at 2048 +# at 2102 +#<date> server id 1 end_log_pos 2102 CRC32 XXX Annotate_rows: #Q> INSERT INTO t2 SELECT * FROM t1 -#<date> server id 1 end_log_pos 2134 CRC32 XXX Table_map: `test`.`t2` mapped to number num -# at 2134 -#<date> server id 1 end_log_pos 2225 CRC32 XXX Write_compressed_rows: table id 33 flags: STMT_END_F +#<date> server id 1 end_log_pos 2164 CRC32 XXX Table_map: `test`.`t2` mapped to number num +# at 2164 +#<date> server id 1 end_log_pos 2255 CRC32 XXX Write_compressed_rows: table id 33 flags: STMT_END_F ### INSERT INTO `test`.`t2` ### SET ### @1=10 /* INT meta=0 nullable=0 is_null=0 */ @@ -223,23 +223,23 @@ BEGIN ### @8=7 /* INT meta=0 nullable=1 is_null=0 */ ### @9='A' /* STRING(1) meta=65025 nullable=1 is_null=0 */ # Number of rows: 4 -# at 2225 -#<date> server id 1 end_log_pos 2298 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 +# at 2255 +#<date> server id 1 end_log_pos 2328 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 SET TIMESTAMP=X/*!*/; COMMIT /*!*/; -# at 2298 -#<date> server id 1 end_log_pos 2340 CRC32 XXX GTID 0-1-8 +# at 2328 +#<date> server id 1 end_log_pos 2370 CRC32 XXX GTID 0-1-8 /*!100001 SET @@session.gtid_seq_no=8*//*!*/; BEGIN /*!*/; -# at 2340 -# at 2406 -#<date> server id 1 end_log_pos 2406 CRC32 XXX Annotate_rows: +# at 2370 +# at 2436 +#<date> server id 1 end_log_pos 2436 CRC32 XXX Annotate_rows: #Q> UPDATE t2 SET f4=5 WHERE f4>0 or f4 is NULL -#<date> server id 1 end_log_pos 2462 CRC32 XXX Table_map: `test`.`t2` mapped to number num -# at 2462 -#<date> server id 1 end_log_pos 2561 CRC32 XXX Update_compressed_rows: table id 33 flags: STMT_END_F +#<date> server id 1 end_log_pos 2498 CRC32 XXX Table_map: `test`.`t2` mapped to number num +# at 2498 +#<date> server id 1 end_log_pos 2597 CRC32 XXX Update_compressed_rows: table id 33 flags: STMT_END_F ### UPDATE `test`.`t2` ### WHERE ### @1=10 /* INT meta=0 nullable=0 is_null=0 */ @@ -304,23 +304,23 @@ BEGIN ### @8=7 /* INT meta=0 nullable=1 is_null=0 */ ### @9='A' /* STRING(1) meta=65025 nullable=1 is_null=0 */ # Number of rows: 3 -# at 2561 -#<date> server id 1 end_log_pos 2634 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 +# at 2597 +#<date> server id 1 end_log_pos 2670 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 SET TIMESTAMP=X/*!*/; COMMIT /*!*/; -# at 2634 -#<date> server id 1 end_log_pos 2676 CRC32 XXX GTID 0-1-9 +# at 2670 +#<date> server id 1 end_log_pos 2712 CRC32 XXX GTID 0-1-9 /*!100001 SET @@session.gtid_seq_no=9*//*!*/; BEGIN /*!*/; -# at 2676 -# at 2713 -#<date> server id 1 end_log_pos 2713 CRC32 XXX Annotate_rows: +# at 2712 +# at 2749 +#<date> server id 1 end_log_pos 2749 CRC32 XXX Annotate_rows: #Q> DELETE FROM t1 -#<date> server id 1 end_log_pos 2769 CRC32 XXX Table_map: `test`.`t1` mapped to number num -# at 2769 -#<date> server id 1 end_log_pos 2861 CRC32 XXX Delete_compressed_rows: table id 32 flags: STMT_END_F +#<date> server id 1 end_log_pos 2811 CRC32 XXX Table_map: `test`.`t1` mapped to number num +# at 2811 +#<date> server id 1 end_log_pos 2903 CRC32 XXX Delete_compressed_rows: table id 32 flags: STMT_END_F ### DELETE FROM `test`.`t1` ### WHERE ### @1=10 /* INT meta=0 nullable=0 is_null=0 */ @@ -366,23 +366,23 @@ BEGIN ### @8=7 /* INT meta=0 nullable=1 is_null=0 */ ### @9='A' /* STRING(1) meta=65025 nullable=1 is_null=0 */ # Number of rows: 4 -# at 2861 -#<date> server id 1 end_log_pos 2934 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 +# at 2903 +#<date> server id 1 end_log_pos 2976 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 SET TIMESTAMP=X/*!*/; COMMIT /*!*/; -# at 2934 -#<date> server id 1 end_log_pos 2976 CRC32 XXX GTID 0-1-10 +# at 2976 +#<date> server id 1 end_log_pos 3018 CRC32 XXX GTID 0-1-10 /*!100001 SET @@session.gtid_seq_no=10*//*!*/; BEGIN /*!*/; -# at 2976 -# at 3013 -#<date> server id 1 end_log_pos 3013 CRC32 XXX Annotate_rows: +# at 3018 +# at 3055 +#<date> server id 1 end_log_pos 3055 CRC32 XXX Annotate_rows: #Q> DELETE FROM t2 -#<date> server id 1 end_log_pos 3069 CRC32 XXX Table_map: `test`.`t2` mapped to number num -# at 3069 -#<date> server id 1 end_log_pos 3154 CRC32 XXX Delete_compressed_rows: table id 33 flags: STMT_END_F +#<date> server id 1 end_log_pos 3117 CRC32 XXX Table_map: `test`.`t2` mapped to number num +# at 3117 +#<date> server id 1 end_log_pos 3202 CRC32 XXX Delete_compressed_rows: table id 33 flags: STMT_END_F ### DELETE FROM `test`.`t2` ### WHERE ### @1=10 /* INT meta=0 nullable=0 is_null=0 */ @@ -428,13 +428,13 @@ BEGIN ### @8=7 /* INT meta=0 nullable=1 is_null=0 */ ### @9='A' /* STRING(1) meta=65025 nullable=1 is_null=0 */ # Number of rows: 4 -# at 3154 -#<date> server id 1 end_log_pos 3227 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 +# at 3202 +#<date> server id 1 end_log_pos 3275 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 SET TIMESTAMP=X/*!*/; COMMIT /*!*/; -# at 3227 -#<date> server id 1 end_log_pos 3275 CRC32 XXX Rotate to master-bin.000002 pos: 4 +# at 3275 +#<date> server id 1 end_log_pos 3323 CRC32 XXX Rotate to master-bin.000002 pos: 4 DELIMITER ; # End of log file ROLLBACK /* added by mysqlbinlog */; diff --git a/mysql-test/main/mysqlbinlog_row_minimal.result b/mysql-test/main/mysqlbinlog_row_minimal.result index b3cee345428..edcefa505a2 100644 --- a/mysql-test/main/mysqlbinlog_row_minimal.result +++ b/mysql-test/main/mysqlbinlog_row_minimal.result @@ -57,9 +57,9 @@ BEGIN # at 890 #<date> server id 1 end_log_pos 890 CRC32 XXX Annotate_rows: #Q> INSERT INTO t1 VALUES (10, 1, 2, 3, 4, 5, 6, 7, "") -#<date> server id 1 end_log_pos 946 CRC32 XXX Table_map: `test`.`t1` mapped to number num -# at 946 -#<date> server id 1 end_log_pos 1015 CRC32 XXX Write_rows: table id 32 flags: STMT_END_F +#<date> server id 1 end_log_pos 952 CRC32 XXX Table_map: `test`.`t1` mapped to number num +# at 952 +#<date> server id 1 end_log_pos 1021 CRC32 XXX Write_rows: table id 32 flags: STMT_END_F ### INSERT INTO `test`.`t1` ### SET ### @1=10 /* INT meta=0 nullable=0 is_null=0 */ @@ -72,23 +72,23 @@ BEGIN ### @8=7 /* INT meta=0 nullable=1 is_null=0 */ ### @9='' /* STRING(1) meta=65025 nullable=1 is_null=0 */ # Number of rows: 1 -# at 1015 -#<date> server id 1 end_log_pos 1088 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 +# at 1021 +#<date> server id 1 end_log_pos 1094 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 SET TIMESTAMP=X/*!*/; COMMIT /*!*/; -# at 1088 -#<date> server id 1 end_log_pos 1130 CRC32 XXX GTID 0-1-4 +# at 1094 +#<date> server id 1 end_log_pos 1136 CRC32 XXX GTID 0-1-4 /*!100001 SET @@session.gtid_seq_no=4*//*!*/; BEGIN /*!*/; -# at 1130 -# at 1206 -#<date> server id 1 end_log_pos 1206 CRC32 XXX Annotate_rows: +# at 1136 +# at 1212 +#<date> server id 1 end_log_pos 1212 CRC32 XXX Annotate_rows: #Q> INSERT INTO t1 VALUES (11, 1, 2, 3, 4, 5, 6, 7, NULL) -#<date> server id 1 end_log_pos 1262 CRC32 XXX Table_map: `test`.`t1` mapped to number num -# at 1262 -#<date> server id 1 end_log_pos 1330 CRC32 XXX Write_rows: table id 32 flags: STMT_END_F +#<date> server id 1 end_log_pos 1274 CRC32 XXX Table_map: `test`.`t1` mapped to number num +# at 1274 +#<date> server id 1 end_log_pos 1342 CRC32 XXX Write_rows: table id 32 flags: STMT_END_F ### INSERT INTO `test`.`t1` ### SET ### @1=11 /* INT meta=0 nullable=0 is_null=0 */ @@ -101,23 +101,23 @@ BEGIN ### @8=7 /* INT meta=0 nullable=1 is_null=0 */ ### @9=NULL /* STRING(1) meta=65025 nullable=1 is_null=1 */ # Number of rows: 1 -# at 1330 -#<date> server id 1 end_log_pos 1403 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 +# at 1342 +#<date> server id 1 end_log_pos 1415 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 SET TIMESTAMP=X/*!*/; COMMIT /*!*/; -# at 1403 -#<date> server id 1 end_log_pos 1445 CRC32 XXX GTID 0-1-5 +# at 1415 +#<date> server id 1 end_log_pos 1457 CRC32 XXX GTID 0-1-5 /*!100001 SET @@session.gtid_seq_no=5*//*!*/; BEGIN /*!*/; -# at 1445 -# at 1523 -#<date> server id 1 end_log_pos 1523 CRC32 XXX Annotate_rows: +# at 1457 +# at 1535 +#<date> server id 1 end_log_pos 1535 CRC32 XXX Annotate_rows: #Q> INSERT INTO t1 VALUES (12, 1, 2, 3, NULL, 5, 6, 7, "A") -#<date> server id 1 end_log_pos 1579 CRC32 XXX Table_map: `test`.`t1` mapped to number num -# at 1579 -#<date> server id 1 end_log_pos 1646 CRC32 XXX Write_rows: table id 32 flags: STMT_END_F +#<date> server id 1 end_log_pos 1597 CRC32 XXX Table_map: `test`.`t1` mapped to number num +# at 1597 +#<date> server id 1 end_log_pos 1664 CRC32 XXX Write_rows: table id 32 flags: STMT_END_F ### INSERT INTO `test`.`t1` ### SET ### @1=12 /* INT meta=0 nullable=0 is_null=0 */ @@ -130,23 +130,23 @@ BEGIN ### @8=7 /* INT meta=0 nullable=1 is_null=0 */ ### @9='A' /* STRING(1) meta=65025 nullable=1 is_null=0 */ # Number of rows: 1 -# at 1646 -#<date> server id 1 end_log_pos 1719 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 +# at 1664 +#<date> server id 1 end_log_pos 1737 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 SET TIMESTAMP=X/*!*/; COMMIT /*!*/; -# at 1719 -#<date> server id 1 end_log_pos 1761 CRC32 XXX GTID 0-1-6 +# at 1737 +#<date> server id 1 end_log_pos 1779 CRC32 XXX GTID 0-1-6 /*!100001 SET @@session.gtid_seq_no=6*//*!*/; BEGIN /*!*/; -# at 1761 -# at 1836 -#<date> server id 1 end_log_pos 1836 CRC32 XXX Annotate_rows: +# at 1779 +# at 1854 +#<date> server id 1 end_log_pos 1854 CRC32 XXX Annotate_rows: #Q> INSERT INTO t1 VALUES (13, 1, 2, 3, 0, 5, 6, 7, "A") -#<date> server id 1 end_log_pos 1892 CRC32 XXX Table_map: `test`.`t1` mapped to number num -# at 1892 -#<date> server id 1 end_log_pos 1962 CRC32 XXX Write_rows: table id 32 flags: STMT_END_F +#<date> server id 1 end_log_pos 1916 CRC32 XXX Table_map: `test`.`t1` mapped to number num +# at 1916 +#<date> server id 1 end_log_pos 1986 CRC32 XXX Write_rows: table id 32 flags: STMT_END_F ### INSERT INTO `test`.`t1` ### SET ### @1=13 /* INT meta=0 nullable=0 is_null=0 */ @@ -159,23 +159,23 @@ BEGIN ### @8=7 /* INT meta=0 nullable=1 is_null=0 */ ### @9='A' /* STRING(1) meta=65025 nullable=1 is_null=0 */ # Number of rows: 1 -# at 1962 -#<date> server id 1 end_log_pos 2035 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 +# at 1986 +#<date> server id 1 end_log_pos 2059 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 SET TIMESTAMP=X/*!*/; COMMIT /*!*/; -# at 2035 -#<date> server id 1 end_log_pos 2077 CRC32 XXX GTID 0-1-7 +# at 2059 +#<date> server id 1 end_log_pos 2101 CRC32 XXX GTID 0-1-7 /*!100001 SET @@session.gtid_seq_no=7*//*!*/; BEGIN /*!*/; -# at 2077 -# at 2131 -#<date> server id 1 end_log_pos 2131 CRC32 XXX Annotate_rows: +# at 2101 +# at 2155 +#<date> server id 1 end_log_pos 2155 CRC32 XXX Annotate_rows: #Q> INSERT INTO t2 SELECT * FROM t1 -#<date> server id 1 end_log_pos 2187 CRC32 XXX Table_map: `test`.`t2` mapped to number num -# at 2187 -#<date> server id 1 end_log_pos 2354 CRC32 XXX Write_rows: table id 33 flags: STMT_END_F +#<date> server id 1 end_log_pos 2217 CRC32 XXX Table_map: `test`.`t2` mapped to number num +# at 2217 +#<date> server id 1 end_log_pos 2384 CRC32 XXX Write_rows: table id 33 flags: STMT_END_F ### INSERT INTO `test`.`t2` ### SET ### @1=10 /* INT meta=0 nullable=0 is_null=0 */ @@ -221,23 +221,23 @@ BEGIN ### @8=7 /* INT meta=0 nullable=1 is_null=0 */ ### @9='A' /* STRING(1) meta=65025 nullable=1 is_null=0 */ # Number of rows: 4 -# at 2354 -#<date> server id 1 end_log_pos 2427 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 +# at 2384 +#<date> server id 1 end_log_pos 2457 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 SET TIMESTAMP=X/*!*/; COMMIT /*!*/; -# at 2427 -#<date> server id 1 end_log_pos 2469 CRC32 XXX GTID 0-1-8 +# at 2457 +#<date> server id 1 end_log_pos 2499 CRC32 XXX GTID 0-1-8 /*!100001 SET @@session.gtid_seq_no=8*//*!*/; BEGIN /*!*/; -# at 2469 -# at 2535 -#<date> server id 1 end_log_pos 2535 CRC32 XXX Annotate_rows: +# at 2499 +# at 2565 +#<date> server id 1 end_log_pos 2565 CRC32 XXX Annotate_rows: #Q> UPDATE t2 SET f4=5 WHERE f4>0 or f4 is NULL -#<date> server id 1 end_log_pos 2591 CRC32 XXX Table_map: `test`.`t2` mapped to number num -# at 2591 -#<date> server id 1 end_log_pos 2657 CRC32 XXX Update_rows: table id 33 flags: STMT_END_F +#<date> server id 1 end_log_pos 2627 CRC32 XXX Table_map: `test`.`t2` mapped to number num +# at 2627 +#<date> server id 1 end_log_pos 2693 CRC32 XXX Update_rows: table id 33 flags: STMT_END_F ### UPDATE `test`.`t2` ### WHERE ### @1=10 /* INT meta=0 nullable=0 is_null=0 */ @@ -254,23 +254,23 @@ BEGIN ### SET ### @5=5 /* INT meta=0 nullable=1 is_null=0 */ # Number of rows: 3 -# at 2657 -#<date> server id 1 end_log_pos 2730 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 +# at 2693 +#<date> server id 1 end_log_pos 2766 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 SET TIMESTAMP=X/*!*/; COMMIT /*!*/; -# at 2730 -#<date> server id 1 end_log_pos 2772 CRC32 XXX GTID 0-1-9 +# at 2766 +#<date> server id 1 end_log_pos 2808 CRC32 XXX GTID 0-1-9 /*!100001 SET @@session.gtid_seq_no=9*//*!*/; BEGIN /*!*/; -# at 2772 -# at 2809 -#<date> server id 1 end_log_pos 2809 CRC32 XXX Annotate_rows: +# at 2808 +# at 2845 +#<date> server id 1 end_log_pos 2845 CRC32 XXX Annotate_rows: #Q> DELETE FROM t1 -#<date> server id 1 end_log_pos 2865 CRC32 XXX Table_map: `test`.`t1` mapped to number num -# at 2865 -#<date> server id 1 end_log_pos 2919 CRC32 XXX Delete_rows: table id 32 flags: STMT_END_F +#<date> server id 1 end_log_pos 2907 CRC32 XXX Table_map: `test`.`t1` mapped to number num +# at 2907 +#<date> server id 1 end_log_pos 2961 CRC32 XXX Delete_rows: table id 32 flags: STMT_END_F ### DELETE FROM `test`.`t1` ### WHERE ### @1=10 /* INT meta=0 nullable=0 is_null=0 */ @@ -284,23 +284,23 @@ BEGIN ### WHERE ### @1=13 /* INT meta=0 nullable=0 is_null=0 */ # Number of rows: 4 -# at 2919 -#<date> server id 1 end_log_pos 2992 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 +# at 2961 +#<date> server id 1 end_log_pos 3034 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 SET TIMESTAMP=X/*!*/; COMMIT /*!*/; -# at 2992 -#<date> server id 1 end_log_pos 3034 CRC32 XXX GTID 0-1-10 +# at 3034 +#<date> server id 1 end_log_pos 3076 CRC32 XXX GTID 0-1-10 /*!100001 SET @@session.gtid_seq_no=10*//*!*/; BEGIN /*!*/; -# at 3034 -# at 3071 -#<date> server id 1 end_log_pos 3071 CRC32 XXX Annotate_rows: +# at 3076 +# at 3113 +#<date> server id 1 end_log_pos 3113 CRC32 XXX Annotate_rows: #Q> DELETE FROM t2 -#<date> server id 1 end_log_pos 3127 CRC32 XXX Table_map: `test`.`t2` mapped to number num -# at 3127 -#<date> server id 1 end_log_pos 3181 CRC32 XXX Delete_rows: table id 33 flags: STMT_END_F +#<date> server id 1 end_log_pos 3175 CRC32 XXX Table_map: `test`.`t2` mapped to number num +# at 3175 +#<date> server id 1 end_log_pos 3229 CRC32 XXX Delete_rows: table id 33 flags: STMT_END_F ### DELETE FROM `test`.`t2` ### WHERE ### @1=10 /* INT meta=0 nullable=0 is_null=0 */ @@ -314,13 +314,13 @@ BEGIN ### WHERE ### @1=13 /* INT meta=0 nullable=0 is_null=0 */ # Number of rows: 4 -# at 3181 -#<date> server id 1 end_log_pos 3254 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 +# at 3229 +#<date> server id 1 end_log_pos 3302 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 SET TIMESTAMP=X/*!*/; COMMIT /*!*/; -# at 3254 -#<date> server id 1 end_log_pos 3302 CRC32 XXX Rotate to master-bin.000002 pos: 4 +# at 3302 +#<date> server id 1 end_log_pos 3350 CRC32 XXX Rotate to master-bin.000002 pos: 4 DELIMITER ; # End of log file ROLLBACK /* added by mysqlbinlog */; @@ -365,9 +365,9 @@ BEGIN #Q> UPDATE t1 t1 INNER JOIN t2 t2 ON t1.ref_id = t2.id #Q> SET t1.is_deleted = TRUE #Q> WHERE t1.id = -#<date> server id 1 end_log_pos 594 CRC32 XXX Table_map: `test`.`t1` mapped to number 35 -# at 594 -#<date> server id 1 end_log_pos 643 CRC32 XXX Update_rows: table id 35 flags: STMT_END_F +#<date> server id 1 end_log_pos 597 CRC32 XXX Table_map: `test`.`t1` mapped to number 35 +# at 597 +#<date> server id 1 end_log_pos 646 CRC32 XXX Update_rows: table id 35 flags: STMT_END_F ### UPDATE `test`.`t1` ### WHERE ### @1=1 /* LONGINT meta=0 nullable=0 is_null=0 */ @@ -375,8 +375,8 @@ BEGIN ### @2=b'1' /* BIT(1) meta=1 nullable=1 is_null=0 */ ### @3=X /* TIMESTAMP(0) meta=0 nullable=0 is_null=0 */ # Number of rows: 1 -# at 643 -#<date> server id 1 end_log_pos 725 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 +# at 646 +#<date> server id 1 end_log_pos 728 CRC32 XXX Query thread_id=5 exec_time=x error_code=0 SET TIMESTAMP=X/*!*/; SET @@session.pseudo_thread_id=5/*!*/; SET @@session.foreign_key_checks=1, @@session.sql_auto_is_null=0, @@session.unique_checks=1, @@session.autocommit=1, @@session.check_constraint_checks=1/*!*/; @@ -388,8 +388,8 @@ SET @@session.lc_time_names=0/*!*/; SET @@session.collation_database=DEFAULT/*!*/; COMMIT /*!*/; -# at 725 -#<date> server id 1 end_log_pos 773 CRC32 XXX Rotate to master-bin.000004 pos: 4 +# at 728 +#<date> server id 1 end_log_pos 776 CRC32 XXX Rotate to master-bin.000004 pos: 4 DELIMITER ; # End of log file ROLLBACK /* added by mysqlbinlog */; diff --git a/mysql-test/main/mysqld--help.result b/mysql-test/main/mysqld--help.result index 3eccd39eac2..ce14d3043c0 100644 --- a/mysql-test/main/mysqld--help.result +++ b/mysql-test/main/mysqld--help.result @@ -114,6 +114,11 @@ The following specify which files/extra groups are read (specified before remain the table) is logged in the before image, and only changed columns are logged in the after image. (Default: FULL). + --binlog-row-metadata=name + Controls whether metadata is logged using FULL or MINIMAL + format. FULL causes all metadata to be logged; MINIMAL + means that only metadata actually required by slave is + logged. Default: MINIMAL. --binlog-stmt-cache-size=# The size of the statement cache for updates to non-transactional engines for the binary log. If you @@ -1427,6 +1432,7 @@ binlog-format MIXED binlog-optimize-thread-scheduling TRUE binlog-row-event-max-size 8192 binlog-row-image FULL +binlog-row-metadata MINIMAL binlog-stmt-cache-size 32768 bulk-insert-buffer-size 8388608 character-set-client-handshake TRUE diff --git a/mysql-test/suite/binlog/include/print_optional_metadata.inc b/mysql-test/suite/binlog/include/print_optional_metadata.inc new file mode 100644 index 00000000000..c91121297ee --- /dev/null +++ b/mysql-test/suite/binlog/include/print_optional_metadata.inc @@ -0,0 +1,32 @@ +# Auxaliary file for printing optional metadata in table_map_log_event +# Usage : +# --let $binlog_file= +# [--let $stop_position] +# [--let $print_primary_key] +# --source extra/binlog_tests/print_optional_metadata.inc + +--let $output_file= $MYSQLTEST_VARDIR/tmp/mysqlbinlog.output + +--let $_stop_position_opt= +if ($stop_position) +{ + --let $_stop_position_opt=--stop-position=$stop_position +} + +--exec $MYSQL_BINLOG -F --print-table-metadata $_stop_position_opt $binlog_file > $output_file + + +--let $grep_pattern= # (?:Columns\(| {8}) +--let $grep_file= $output_file +--source include/grep_pattern.inc + +if ($print_primary_key) +{ + --let $grep_pattern= # Primary Key + --source include/grep_pattern.inc +} +--remove_file $output_file +--let $stop_position= +--let $_stop_position_opt= + + diff --git a/mysql-test/suite/binlog/r/binlog_table_map_optional_metadata.result b/mysql-test/suite/binlog/r/binlog_table_map_optional_metadata.result new file mode 100644 index 00000000000..c478ae5a3f1 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_table_map_optional_metadata.result @@ -0,0 +1,401 @@ +RESET MASTER; +# +# Temporal types can be printed correctly +# +CREATE TABLE t1(c_year YEAR, c_date DATE, c_time TIME, c_time_f TIME(3), +c_datetime DATETIME, c_datetime_f DATETIME(3), +c_timestamp TIMESTAMP, c_timestamp_f TIMESTAMP(3) DEFAULT "2017-1-1 10:10:10"); +INSERT INTO t1(c_year) VALUES(2017); +Matching lines are: +# Columns(YEAR, +# DATE, +# TIME, +# TIME(3), +# DATETIME, +# DATETIME(3), +# TIMESTAMP NOT NULL, +# TIMESTAMP(3) NOT NULL) +Occurrences of '# (?:Columns\(| {8})' in the input file: 8 +DROP TABLE t1; +RESET MASTER; +# +# Json types can be printed correctly +# +CREATE TABLE t1 (c_json JSON, c_char CHAR(100)); +INSERT INTO t1(c_char) VALUES("abc"); +Matching lines are: +# Columns(LONGTEXT CHARSET utf8mb4 COLLATE utf8mb4_bin CHARSET utf8mb4 COLLATE utf8mb4_bin, +# CHAR(100) CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci) +Occurrences of '# (?:Columns\(| {8})' in the input file: 2 +DROP TABLE t1; +RESET MASTER; +# +# Geometry types can be printed correctly +# +CREATE TABLE t1 (c_geo GEOMETRY, c_point POINT, c_linestring LINESTRING, +c_polygon POLYGON, c_multi_point MULTIPOINT, +c_multi_linestring MULTILINESTRING, c_multi_polygon MULTIPOLYGON, +c_geometrycollection GEOMETRYCOLLECTION, c_char CHAR(100)); +INSERT INTO t1(c_point) VALUES(ST_PointFromText('POINT(10 10)')); +Matching lines are: +# Columns(GEOMETRY, +# POINT, +# LINESTRING, +# POLYGON, +# MULTIPOINT, +# MULTILINESTRING, +# MULTIPOLYGON, +# GEOMETRYCOLLECTION, +# CHAR(100) CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci) +Occurrences of '# (?:Columns\(| {8})' in the input file: 9 +RESET MASTER; +SET GLOBAL binlog_row_metadata = FULL; +INSERT INTO t1(c_point) VALUES(ST_PointFromText('POINT(10 10)')); +Matching lines are: +# Columns(`c_geo` GEOMETRY, +# `c_point` POINT, +# `c_linestring` LINESTRING, +# `c_polygon` POLYGON, +# `c_multi_point` MULTIPOINT, +# `c_multi_linestring` MULTILINESTRING, +# `c_multi_polygon` MULTIPOLYGON, +# `c_geometrycollection` GEOMETRYCOLLECTION, +# `c_char` CHAR(100) CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci) +Occurrences of '# (?:Columns\(| {8})' in the input file: 9 +DROP TABLE t1; +RESET MASTER; +# +# Numeric types can be printed correctly +# +CREATE TABLE t1(c_bit BIT(10), c_bool BOOL, c_smallint SMALLINT, +c_mediumint MEDIUMINT, c_int INT UNSIGNED, c_bigint BIGINT, +c_float FLOAT UNSIGNED, c_double DOUBLE, c_decimal DECIMAL(10, 2)); +SET GLOBAL binlog_row_metadata = MINIMAL; +INSERT INTO t1(c_bool) VALUES(1); +# UNSIGNED flag should be printed +Matching lines are: +# Columns(BIT(10), +# TINYINT, +# SMALLINT, +# MEDIUMINT, +# INT UNSIGNED, +# BIGINT, +# FLOAT UNSIGNED, +# DOUBLE, +# DECIMAL(10,2)) +Occurrences of '# (?:Columns\(| {8})' in the input file: 9 +RESET MASTER; +SET GLOBAL binlog_row_metadata = FULL; +INSERT INTO t1(c_bool) VALUES(1); +Matching lines are: +# Columns(`c_bit` BIT(10), +# `c_bool` TINYINT, +# `c_smallint` SMALLINT, +# `c_mediumint` MEDIUMINT, +# `c_int` INT UNSIGNED, +# `c_bigint` BIGINT, +# `c_float` FLOAT UNSIGNED, +# `c_double` DOUBLE, +# `c_decimal` DECIMAL(10,2)) +Occurrences of '# (?:Columns\(| {8})' in the input file: 9 +DROP TABLE t1; +RESET MASTER; +# +# Character types can be printed correctly +# +CREATE TABLE t1(c_char CHAR(10), c_varchar VARCHAR(500), +c_tinytext TINYTEXT, c_text TEXT, +c_mediumtext MEDIUMTEXT, c_longtext LONGTEXT CHARSET utf8); +SET GLOBAL binlog_row_metadata = MINIMAL; +INSERT INTO t1(c_char) VALUES("1"); +Matching lines are: +# Columns(CHAR(10) CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# VARCHAR(500) CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# TINYTEXT CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# TEXT CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# MEDIUMTEXT CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# LONGTEXT CHARSET utf8 COLLATE utf8_general_ci CHARSET utf8 COLLATE utf8_general_ci) +Occurrences of '# (?:Columns\(| {8})' in the input file: 6 +RESET MASTER; +SET GLOBAL binlog_row_metadata = FULL; +INSERT INTO t1(c_char) VALUES("1"); +Matching lines are: +# Columns(`c_char` CHAR(10) CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# `c_varchar` VARCHAR(500) CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# `c_tinytext` TINYTEXT CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# `c_text` TEXT CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# `c_mediumtext` MEDIUMTEXT CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# `c_longtext` LONGTEXT CHARSET utf8 COLLATE utf8_general_ci CHARSET utf8 COLLATE utf8_general_ci) +Occurrences of '# (?:Columns\(| {8})' in the input file: 6 +DROP TABLE t1; +RESET MASTER; +# +# Column names with non-ascii characters and escape characters can be printed correctly +# +CREATE TABLE t1(`åäö表\a'``"` INT); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `åäö表\a'``"` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +INSERT INTO t1 VALUES(1); +Matching lines are: +# Columns(`åäö表\\a\'`"` INT) +Occurrences of '# (?:Columns\(| {8})' in the input file: 1 +DROP TABLE t1; +RESET MASTER; +# +# Charsets can be printed correctly +# +CREATE TABLE t1(c_char_utf8 CHAR(10) CHARSET utf8, +c_varchar_utf8 VARCHAR(10) CHARSET utf8, +c_text_utf8 TEXT CHARSET utf8); +INSERT INTO t1 VALUES("1", "2", "3"); +Matching lines are: +# Columns(`c_char_utf8` CHAR(10) CHARSET utf8 COLLATE utf8_general_ci CHARSET utf8 COLLATE utf8_general_ci, +# `c_varchar_utf8` VARCHAR(10) CHARSET utf8 COLLATE utf8_general_ci CHARSET utf8 COLLATE utf8_general_ci, +# `c_text_utf8` TEXT CHARSET utf8 COLLATE utf8_general_ci CHARSET utf8 COLLATE utf8_general_ci) +Occurrences of '# (?:Columns\(| {8})' in the input file: 3 +DROP TABLE t1; +RESET MASTER; +CREATE TABLE t1(c_utf8mb4_520 CHAR(10) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci, +c_utf8mb4_0900 VARCHAR(10) CHARSET utf8mb4 COLLATE utf8mb4_polish_ci, +c_utf8mb4_def TEXT CHARSET utf8mb4); +INSERT INTO t1 VALUES("1", "2", "3"); +Matching lines are: +# Columns(`c_utf8mb4_520` CHAR(10) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci, +# `c_utf8mb4_0900` VARCHAR(10) CHARSET utf8mb4 COLLATE utf8mb4_polish_ci CHARSET utf8mb4 COLLATE utf8mb4_polish_ci, +# `c_utf8mb4_def` TEXT CHARSET utf8mb4 COLLATE utf8mb4_general_ci CHARSET utf8mb4 COLLATE utf8mb4_general_ci) +Occurrences of '# (?:Columns\(| {8})' in the input file: 3 +DROP TABLE t1; +RESET MASTER; +# +# Blob and binary columns can be printed correctly +# +CREATE TABLE t1(c_binary BINARY(10), c_varbinary VARBINARY(10), +c_tinyblob TINYBLOB, c_blob BLOB, +c_mediumblob MEDIUMBLOB, c_longblob LONGBLOB); +INSERT INTO t1 VALUES("1", "2", "3", "4", "5", "6"); +Matching lines are: +# Columns(`c_binary` BINARY(10), +# `c_varbinary` VARBINARY(10), +# `c_tinyblob` TINYBLOB, +# `c_blob` BLOB, +# `c_mediumblob` MEDIUMBLOB, +# `c_longblob` LONGBLOB) +Occurrences of '# (?:Columns\(| {8})' in the input file: 6 +DROP TABLE t1; +RESET MASTER; +# +# Verify that SET string values and character sets can be printed correctly +# +CREATE TABLE t1( +c_set_1 SET("set1_v1_å", "set1_v2_ä", "set1_v3_ö"), +c_set_2 SET("set2_v1_å", "set2_v2_ä", "set2_v3_ö") CHARACTER SET binary, +c_set_3 SET("set3_v1_å", "set3_v2_ä", "set3_v3_ö") CHARACTER SET latin1, +c_set_4 SET("set4_v1_å", "set4_v2_ä", "set4_v3_ö") CHARACTER SET swe7 COLLATE swe7_bin, +c_set_5 SET("set5_v1_å", "set5_v2_ä", "set5_v3_ö") CHARACTER SET ucs2, +c_set_6 SET("set6_v1_å", "set6_v2_ä", "set6_v3_ö") CHARACTER SET utf32); +SET GLOBAL binlog_row_metadata = MINIMAL; +INSERT INTO t1 VALUES("set1_v1_å", "set2_v2_ä", "set3_v3_ö", "set4_v1_å", "set5_v2_ä", "set6_v3_ö"); +Matching lines are: +# Columns(SET, +# SET, +# SET, +# SET, +# SET, +# SET) +Occurrences of '# (?:Columns\(| {8})' in the input file: 6 +RESET MASTER; +SET GLOBAL binlog_row_metadata = FULL; +INSERT INTO t1 VALUES("set1_v1_å", "set2_v2_ä", "set3_v3_ö", "set4_v1_å", "set5_v2_ä", "set6_v3_ö"); +Matching lines are: +# Columns(`c_set_1` SET('set1_v1_å','set1_v2_ä','set1_v3_ö') CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# `c_set_2` SET('set2_v1_å','set2_v2_ä','set2_v3_ö') CHARSET binary COLLATE binary CHARSET binary COLLATE binary, +# `c_set_3` SET('set3_v1_å','set3_v2_ä','set3_v3_ö') CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# `c_set_4` SET('set4_v1_??','set4_v2_??','set4_v3_??') CHARSET swe7 COLLATE swe7_bin CHARSET swe7 COLLATE swe7_bin, +# `c_set_5` SET('\0s\0e\0t\05\0_\0v\01\0_\0�\0�','\0s\0e\0t\05\0_\0v\02\0_\0�\0�','\0s\0e\0t\05\0_\0v\03\0_\0�\0�') CHARSET ucs2 COLLATE ucs2_general_ci CHARSET ucs2 COLLATE ucs2_general_ci, +# `c_set_6` SET('\0\0\0s\0\0\0e\0\0\0t\0\0\06\0\0\0_\0\0\0v\0\0\01\0\0\0_\0\0\0�\0\0\0�','\0\0\0s\0\0\0e\0\0\0t\0\0\06\0\0\0_\0\0\0v\0\0\02\0\0\0_\0\0\0�\0\0\0�','\0\0\0s\0\0\0e\0\0\0t\0\0\06\0\0\0_\0\0\0v\0\0\03\0\0\0_\0\0\0�\0\0\0�') CHARSET utf32 COLLATE utf32_general_ci CHARSET utf32 COLLATE utf32_general_ci) +Occurrences of '# (?:Columns\(| {8})' in the input file: 6 +DROP TABLE t1; +RESET MASTER; +# +# Verify that ENUM string values and character sets can be printed correctly +# +CREATE TABLE t1( +c_enum_1 ENUM("enum1_v1_å", "enum1_v2_ä", "enum1_v3_ö"), +c_enum_2 ENUM("enum2_v1_å", "enum2_v2_ä", "enum2_v3_ö") CHARACTER SET binary, +c_enum_3 ENUM("enum3_v1_å", "enum3_v2_ä", "enum3_v3_ö") CHARACTER SET latin1, +c_enum_4 ENUM("enum4_v1_å", "enum4_v2_ä", "enum4_v3_ö") CHARACTER SET swe7 COLLATE swe7_bin, +c_enum_5 ENUM("enum5_v1_å", "enum5_v2_ä", "enum5_v3_ö") CHARACTER SET ucs2, +c_enum_6 ENUM("enum6_v1_å", "enum6_v2_ä", "enum6_v3_ö") CHARACTER SET utf32 +); +SET GLOBAL binlog_row_metadata = MINIMAL; +INSERT INTO t1 VALUES("enum1_v1_å", "enum2_v2_ä", "enum3_v3_ö", "enum4_v1_å", "enum5_v2_ä", "enum6_v3_ö"); +Matching lines are: +# Columns(ENUM, +# ENUM, +# ENUM, +# ENUM, +# ENUM, +# ENUM) +Occurrences of '# (?:Columns\(| {8})' in the input file: 6 +RESET MASTER; +SET GLOBAL binlog_row_metadata = FULL; +INSERT INTO t1 VALUES("enum1_v1_å", "enum2_v2_ä", "enum3_v3_ö", "enum4_v1_å", "enum5_v2_ä", "enum6_v3_ö"); +Matching lines are: +# Columns(`c_enum_1` ENUM('enum1_v1_å','enum1_v2_ä','enum1_v3_ö') CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# `c_enum_2` ENUM('enum2_v1_å','enum2_v2_ä','enum2_v3_ö') CHARSET binary COLLATE binary CHARSET binary COLLATE binary, +# `c_enum_3` ENUM('enum3_v1_å','enum3_v2_ä','enum3_v3_ö') CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# `c_enum_4` ENUM('enum4_v1_??','enum4_v2_??','enum4_v3_??') CHARSET swe7 COLLATE swe7_bin CHARSET swe7 COLLATE swe7_bin, +# `c_enum_5` ENUM('\0e\0n\0u\0m\05\0_\0v\01\0_\0�\0�','\0e\0n\0u\0m\05\0_\0v\02\0_\0�\0�','\0e\0n\0u\0m\05\0_\0v\03\0_\0�\0�') CHARSET ucs2 COLLATE ucs2_general_ci CHARSET ucs2 COLLATE ucs2_general_ci, +# `c_enum_6` ENUM('\0\0\0e\0\0\0n\0\0\0u\0\0\0m\0\0\06\0\0\0_\0\0\0v\0\0\01\0\0\0_\0\0\0�\0\0\0�','\0\0\0e\0\0\0n\0\0\0u\0\0\0m\0\0\06\0\0\0_\0\0\0v\0\0\02\0\0\0_\0\0\0�\0\0\0�','\0\0\0e\0\0\0n\0\0\0u\0\0\0m\0\0\06\0\0\0_\0\0\0v\0\0\03\0\0\0_\0\0\0�\0\0\0�') CHARSET utf32 COLLATE utf32_general_ci CHARSET utf32 COLLATE utf32_general_ci) +Occurrences of '# (?:Columns\(| {8})' in the input file: 6 +DROP TABLE t1; +RESET MASTER; +# +# Verify that explicit NOT NULL can be printed correctly +# +CREATE TABLE t1(c_not_null1 INT NOT NULL, c_null1 INT, c_not_null2 INT NOT NULL, +c_null2 INT); +INSERT INTO t1 VALUES(1, 2, 3, 4); +Matching lines are: +# Columns(`c_not_null1` INT NOT NULL, +# `c_null1` INT, +# `c_not_null2` INT NOT NULL, +# `c_null2` INT) +Occurrences of '# (?:Columns\(| {8})' in the input file: 4 +DROP TABLE t1; +RESET MASTER; +# +# Verify that primary key can be printed correctly +# +CREATE TABLE t1(c_key1 INT, c_key3 INT, c_not_key INT, c_key2 INT, +PRIMARY KEY(c_key1, c_key2, c_key3)); +INSERT INTO t1 VALUES(1, 2, 3, 4); +Matching lines are: +# Columns(`c_key1` INT NOT NULL, +# `c_key3` INT NOT NULL, +# `c_not_key` INT, +# `c_key2` INT NOT NULL) +Occurrences of '# (?:Columns\(| {8})' in the input file: 4 +Matching lines are: +# Primary Key(c_key1, c_key2, c_key3) +Occurrences of '# Primary Key' in the input file: 1 +DROP TABLE t1; +RESET MASTER; +CREATE TABLE t1(c_key1 CHAR(100), c_key3 CHAR(100), c_not_key INT, c_key2 CHAR(10), +PRIMARY KEY(c_key1(5), c_key2, c_key3(10))); +INSERT INTO t1 VALUES("1", "2", 3, "4"); +Matching lines are: +# Columns(`c_key1` CHAR(100) NOT NULL CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# `c_key3` CHAR(100) NOT NULL CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# `c_not_key` INT, +# `c_key2` CHAR(10) NOT NULL CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci) +Occurrences of '# (?:Columns\(| {8})' in the input file: 4 +Matching lines are: +# Primary Key(c_key1(5), c_key2, c_key3(10)) +Occurrences of '# Primary Key' in the input file: 1 +RESET MASTER; +SET GLOBAL binlog_row_metadata = MINIMAL; +INSERT INTO t1 VALUES("2", "2", 3, "4"); +Matching lines are: +# Columns(CHAR(100) NOT NULL CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# CHAR(100) NOT NULL CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# INT, +# CHAR(10) NOT NULL CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci) +Occurrences of '# (?:Columns\(| {8})' in the input file: 4 +Matching lines are: +None +Occurrences of '# Primary Key' in the input file: 0 +RESET MASTER; +# +# Coverage test: Print column index instead of column name if column name +# is not binlogged. +# +SET GLOBAL binlog_row_metadata = FULL; +SET SESSION debug_dbug = 'd, dont_log_column_name'; +INSERT INTO t1 VALUES("3", "2", 3, "4"); +Matching lines are: +# Columns(`c_key1` CHAR(100) NOT NULL CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# `c_key3` CHAR(100) NOT NULL CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci, +# `c_not_key` INT, +# `c_key2` CHAR(10) NOT NULL CHARSET latin1 COLLATE latin1_swedish_ci CHARSET latin1 COLLATE latin1_swedish_ci) +Occurrences of '# (?:Columns\(| {8})' in the input file: 4 +Matching lines are: +# Primary Key(c_key1(5), c_key2, c_key3(10)) +Occurrences of '# Primary Key' in the input file: 1 +DROP TABLE t1; +RESET MASTER; +# +# Coverage test: Inject an invalid column type +# +CREATE TABLE t1(c1 int, c2 BLOB); +SET SESSION debug_dbug = 'd,inject_invalid_column_type'; +INSERT INTO t1 VALUES(1, "a"); +Matching lines are: +# Columns(`c1` INT, +# `c2` INVALID_TYPE(230)) +Occurrences of '# (?:Columns\(| {8})' in the input file: 2 +RESET MASTER; +# +# Coverage test: Inject an invalid BLOB metadata +# +SET SESSION debug_dbug = 'd,inject_invalid_blob_size'; +INSERT INTO t1 VALUES(2, "b"); +Matching lines are: +# Columns(`c1` INT, +# `c2` INVALID_BLOB(5)) +Occurrences of '# (?:Columns\(| {8})' in the input file: 2 +# +# Coverage test: Inject an invalid Geometry type +# +DROP TABLE t1; +CREATE TABLE t1(c_geometry GEOMETRY, c_point POINT, c_multilinestring MULTILINESTRING); +RESET MASTER; +SET SESSION debug_dbug = 'd,inject_invalid_geometry_type'; +INSERT INTO t1(c_point) VALUES(ST_PointFromText('POINT(10 10)')); +Matching lines are: +# Columns(`c_geometry` INVALID_GEOMETRY_TYPE(100), +# `c_point` INVALID_GEOMETRY_TYPE(100), +# `c_multilinestring` INVALID_GEOMETRY_TYPE(100)) +Occurrences of '# (?:Columns\(| {8})' in the input file: 3 +DROP TABLE t1; +RESET MASTER; +# +# Comptibility Test: Verify mysqlbinlog can print OLD table_map_log_event +# without any optional metadata +# +CREATE TABLE t1(c_int INT, c_tiny_int_unsigned TINYINT UNSIGNED, +c_binary BINARY(10), c_text TEXT, c_point POINT); +SET session debug_dbug='d,simulate_no_optional_metadata'; +INSERT INTO t1(c_int) VALUES(1); +Matching lines are: +# Columns(INT, +# TINYINT, +# BINARY(10), +# BLOB, +# GEOMETRY) +Occurrences of '# (?:Columns\(| {8})' in the input file: 5 +DROP TABLE t1; +RESET MASTER; +# +# Simulate error on initializing charset and primary key metadata +# +CREATE TABLE t1(c1 char(10) PRIMARY KEY); +SET session debug_dbug='d,simulate_init_charset_field_error'; +INSERT INTO t1 VALUES("a"); +SET GLOBAL binlog_row_metadata = FULL; +SET session debug_dbug='d,simulate_init_primary_key_field_error'; +INSERT INTO t1 VALUES("b"); +Matching lines are: +# Columns(BINARY(10) NOT NULL) +# Columns(BINARY(10) NOT NULL) +Occurrences of '# (?:Columns\(| {8})' in the input file: 2 +Matching lines are: +None +Occurrences of '# Primary Key' in the input file: 0 +SET SESSION debug_dbug = ''; +SET GLOBAL binlog_row_metadata = MINIMAL; +DROP TABLE t1; +RESET MASTER; diff --git a/mysql-test/suite/binlog/t/binlog_table_map_optional_metadata.test b/mysql-test/suite/binlog/t/binlog_table_map_optional_metadata.test new file mode 100644 index 00000000000..be891759a7e --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_table_map_optional_metadata.test @@ -0,0 +1,348 @@ +################################################################################ +# WL#4618 RBR: extended table metadata in the binary log +# +# Below metadata is logged into Table_map_log_event +# - signedness of numeric columns +# - charsets of character columns +# - column names +# - set/enum character sets and string values +# - primary key +# +# The first two are always logged. The others are controlled by system +# variable --binlog-row-metadata +# +# The test will verify if the metadata can be logged and printed by mysqlbinlog +# correctly. +# mysqlbinlog --print-table-metadata will print the extra metadata +################################################################################ +--source include/have_debug.inc +--source include/have_binlog_format_row.inc + +RESET MASTER; + +--let $MYSQLD_DATADIR= `select @@datadir` +--let $binlog_file= $MYSQLD_DATADIR/master-bin.000001 + +--echo # +--echo # Temporal types can be printed correctly +--echo # +CREATE TABLE t1(c_year YEAR, c_date DATE, c_time TIME, c_time_f TIME(3), + c_datetime DATETIME, c_datetime_f DATETIME(3), + c_timestamp TIMESTAMP, c_timestamp_f TIMESTAMP(3) DEFAULT "2017-1-1 10:10:10"); + +INSERT INTO t1(c_year) VALUES(2017); +--source include/print_optional_metadata.inc + +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Json types can be printed correctly +--echo # +CREATE TABLE t1 (c_json JSON, c_char CHAR(100)); +INSERT INTO t1(c_char) VALUES("abc"); +--source include/print_optional_metadata.inc + +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Geometry types can be printed correctly +--echo # +CREATE TABLE t1 (c_geo GEOMETRY, c_point POINT, c_linestring LINESTRING, + c_polygon POLYGON, c_multi_point MULTIPOINT, + c_multi_linestring MULTILINESTRING, c_multi_polygon MULTIPOLYGON, + c_geometrycollection GEOMETRYCOLLECTION, c_char CHAR(100)); + +INSERT INTO t1(c_point) VALUES(ST_PointFromText('POINT(10 10)')); +--source include/print_optional_metadata.inc + +RESET MASTER; +SET GLOBAL binlog_row_metadata = FULL; + +# geometry type is binlogged, the real geometry types are printed +INSERT INTO t1(c_point) VALUES(ST_PointFromText('POINT(10 10)')); +--source include/print_optional_metadata.inc + +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Numeric types can be printed correctly +--echo # +CREATE TABLE t1(c_bit BIT(10), c_bool BOOL, c_smallint SMALLINT, + c_mediumint MEDIUMINT, c_int INT UNSIGNED, c_bigint BIGINT, + c_float FLOAT UNSIGNED, c_double DOUBLE, c_decimal DECIMAL(10, 2)); + +SET GLOBAL binlog_row_metadata = MINIMAL; +INSERT INTO t1(c_bool) VALUES(1); + +--echo # UNSIGNED flag should be printed +--source include/print_optional_metadata.inc + +RESET MASTER; +SET GLOBAL binlog_row_metadata = FULL; +INSERT INTO t1(c_bool) VALUES(1); + +--source include/print_optional_metadata.inc + +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Character types can be printed correctly +--echo # +CREATE TABLE t1(c_char CHAR(10), c_varchar VARCHAR(500), + c_tinytext TINYTEXT, c_text TEXT, + c_mediumtext MEDIUMTEXT, c_longtext LONGTEXT CHARSET utf8); + +SET GLOBAL binlog_row_metadata = MINIMAL; +INSERT INTO t1(c_char) VALUES("1"); + +# Charset set is printed with default charset +--source include/print_optional_metadata.inc + +RESET MASTER; +SET GLOBAL binlog_row_metadata = FULL; +INSERT INTO t1(c_char) VALUES("1"); + +--source include/print_optional_metadata.inc + +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Column names with non-ascii characters and escape characters can be printed correctly +--echo # +CREATE TABLE t1(`åäö表\a'``"` INT); + +SHOW CREATE TABLE t1; + +INSERT INTO t1 VALUES(1); +--source include/print_optional_metadata.inc +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Charsets can be printed correctly +--echo # +CREATE TABLE t1(c_char_utf8 CHAR(10) CHARSET utf8, + c_varchar_utf8 VARCHAR(10) CHARSET utf8, + c_text_utf8 TEXT CHARSET utf8); + +INSERT INTO t1 VALUES("1", "2", "3"); + +# Charset set is printed with Default charset +--source include/print_optional_metadata.inc + +DROP TABLE t1; +RESET MASTER; + +# Test collation number less than 250 and collation number greater than 250 +CREATE TABLE t1(c_utf8mb4_520 CHAR(10) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci, + c_utf8mb4_0900 VARCHAR(10) CHARSET utf8mb4 COLLATE utf8mb4_polish_ci, + c_utf8mb4_def TEXT CHARSET utf8mb4); + +INSERT INTO t1 VALUES("1", "2", "3"); + +# Charset set is printed without default charset +--source include/print_optional_metadata.inc + +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Blob and binary columns can be printed correctly +--echo # +CREATE TABLE t1(c_binary BINARY(10), c_varbinary VARBINARY(10), + c_tinyblob TINYBLOB, c_blob BLOB, + c_mediumblob MEDIUMBLOB, c_longblob LONGBLOB); + +INSERT INTO t1 VALUES("1", "2", "3", "4", "5", "6"); +--source include/print_optional_metadata.inc + +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Verify that SET string values and character sets can be printed correctly +--echo # + +CREATE TABLE t1( + c_set_1 SET("set1_v1_å", "set1_v2_ä", "set1_v3_ö"), + c_set_2 SET("set2_v1_å", "set2_v2_ä", "set2_v3_ö") CHARACTER SET binary, + c_set_3 SET("set3_v1_å", "set3_v2_ä", "set3_v3_ö") CHARACTER SET latin1, + c_set_4 SET("set4_v1_å", "set4_v2_ä", "set4_v3_ö") CHARACTER SET swe7 COLLATE swe7_bin, + c_set_5 SET("set5_v1_å", "set5_v2_ä", "set5_v3_ö") CHARACTER SET ucs2, + c_set_6 SET("set6_v1_å", "set6_v2_ä", "set6_v3_ö") CHARACTER SET utf32); + +SET GLOBAL binlog_row_metadata = MINIMAL; +INSERT INTO t1 VALUES("set1_v1_å", "set2_v2_ä", "set3_v3_ö", "set4_v1_å", "set5_v2_ä", "set6_v3_ö"); +--source include/print_optional_metadata.inc + +RESET MASTER; +SET GLOBAL binlog_row_metadata = FULL; +INSERT INTO t1 VALUES("set1_v1_å", "set2_v2_ä", "set3_v3_ö", "set4_v1_å", "set5_v2_ä", "set6_v3_ö"); +--source include/print_optional_metadata.inc + +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Verify that ENUM string values and character sets can be printed correctly +--echo # + +CREATE TABLE t1( + c_enum_1 ENUM("enum1_v1_å", "enum1_v2_ä", "enum1_v3_ö"), + c_enum_2 ENUM("enum2_v1_å", "enum2_v2_ä", "enum2_v3_ö") CHARACTER SET binary, + c_enum_3 ENUM("enum3_v1_å", "enum3_v2_ä", "enum3_v3_ö") CHARACTER SET latin1, + c_enum_4 ENUM("enum4_v1_å", "enum4_v2_ä", "enum4_v3_ö") CHARACTER SET swe7 COLLATE swe7_bin, + c_enum_5 ENUM("enum5_v1_å", "enum5_v2_ä", "enum5_v3_ö") CHARACTER SET ucs2, + c_enum_6 ENUM("enum6_v1_å", "enum6_v2_ä", "enum6_v3_ö") CHARACTER SET utf32 + ); + +SET GLOBAL binlog_row_metadata = MINIMAL; +INSERT INTO t1 VALUES("enum1_v1_å", "enum2_v2_ä", "enum3_v3_ö", "enum4_v1_å", "enum5_v2_ä", "enum6_v3_ö"); +--source include/print_optional_metadata.inc + +RESET MASTER; +SET GLOBAL binlog_row_metadata = FULL; +INSERT INTO t1 VALUES("enum1_v1_å", "enum2_v2_ä", "enum3_v3_ö", "enum4_v1_å", "enum5_v2_ä", "enum6_v3_ö"); +--source include/print_optional_metadata.inc + +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Verify that explicit NOT NULL can be printed correctly +--echo # +CREATE TABLE t1(c_not_null1 INT NOT NULL, c_null1 INT, c_not_null2 INT NOT NULL, + c_null2 INT); + +INSERT INTO t1 VALUES(1, 2, 3, 4); +--source include/print_optional_metadata.inc + +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Verify that primary key can be printed correctly +--echo # +CREATE TABLE t1(c_key1 INT, c_key3 INT, c_not_key INT, c_key2 INT, +PRIMARY KEY(c_key1, c_key2, c_key3)); + +INSERT INTO t1 VALUES(1, 2, 3, 4); +--let $print_primary_key= 1 +--source include/print_optional_metadata.inc + +DROP TABLE t1; +RESET MASTER; + +# Key has prefix +CREATE TABLE t1(c_key1 CHAR(100), c_key3 CHAR(100), c_not_key INT, c_key2 CHAR(10), +PRIMARY KEY(c_key1(5), c_key2, c_key3(10))); + +INSERT INTO t1 VALUES("1", "2", 3, "4"); +--source include/print_optional_metadata.inc + +RESET MASTER; +# Primary key should not be printed +SET GLOBAL binlog_row_metadata = MINIMAL; + +INSERT INTO t1 VALUES("2", "2", 3, "4"); +--source include/print_optional_metadata.inc + +RESET MASTER; +--echo # +--echo # Coverage test: Print column index instead of column name if column name +--echo # is not binlogged. +--echo # +SET GLOBAL binlog_row_metadata = FULL; + +SET SESSION debug_dbug = 'd, dont_log_column_name'; +INSERT INTO t1 VALUES("3", "2", 3, "4"); +--source include/print_optional_metadata.inc + +--let $print_primary_key= +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Coverage test: Inject an invalid column type +--echo # +CREATE TABLE t1(c1 int, c2 BLOB); + +SET SESSION debug_dbug = 'd,inject_invalid_column_type'; +INSERT INTO t1 VALUES(1, "a"); +# It prints an error +--source include/print_optional_metadata.inc + +RESET MASTER; + +--echo # +--echo # Coverage test: Inject an invalid BLOB metadata +--echo # +--let $start_pos= query_get_value(SHOW MASTER STATUS, Position, 1) + +SET SESSION debug_dbug = 'd,inject_invalid_blob_size'; +INSERT INTO t1 VALUES(2, "b"); + +# The invalid metadata will case assertion failure on Write_rows_log_event +# So we need to stop mysqlbinlog before reading Write_rows_log_event. +--let $stop_position= query_get_value(SHOW BINLOG EVENTS FROM $start_pos LIMIT 3, End_log_pos, 3) +--source include/print_optional_metadata.inc + +--echo # +--echo # Coverage test: Inject an invalid Geometry type +--echo # +DROP TABLE t1; +CREATE TABLE t1(c_geometry GEOMETRY, c_point POINT, c_multilinestring MULTILINESTRING); +RESET MASTER; +--let $start_pos= query_get_value(SHOW MASTER STATUS, Position, 1) + +SET SESSION debug_dbug = 'd,inject_invalid_geometry_type'; +INSERT INTO t1(c_point) VALUES(ST_PointFromText('POINT(10 10)')); + +# The invalid metadata will case assertion failure on Write_rows_log_event +# So we need to stop mysqlbinlog before reading Write_rows_log_event. +--let $stop_position= query_get_value(SHOW BINLOG EVENTS FROM $start_pos LIMIT 3, End_log_pos, 3) +--source include/print_optional_metadata.inc + +DROP TABLE t1; +RESET MASTER; +--echo # +--echo # Comptibility Test: Verify mysqlbinlog can print OLD table_map_log_event +--echo # without any optional metadata +--echo # +CREATE TABLE t1(c_int INT, c_tiny_int_unsigned TINYINT UNSIGNED, + c_binary BINARY(10), c_text TEXT, c_point POINT); + +SET session debug_dbug='d,simulate_no_optional_metadata'; +INSERT INTO t1(c_int) VALUES(1); +# TINYINT will be printed without UNSIGNED flag, +# CHAR will be printed as BINARY(10) +# POINT will be printed as GEOMETRY +--let $stop_position= +--source include/print_optional_metadata.inc + +DROP TABLE t1; +RESET MASTER; +--echo # +--echo # Simulate error on initializing charset and primary key metadata +--echo # +CREATE TABLE t1(c1 char(10) PRIMARY KEY); + +SET session debug_dbug='d,simulate_init_charset_field_error'; +INSERT INTO t1 VALUES("a"); + +SET GLOBAL binlog_row_metadata = FULL; +SET session debug_dbug='d,simulate_init_primary_key_field_error'; +INSERT INTO t1 VALUES("b"); + +--let $print_primary_key= 1 +--source include/print_optional_metadata.inc + +SET SESSION debug_dbug = ''; +SET GLOBAL binlog_row_metadata = MINIMAL; +DROP TABLE t1; +RESET MASTER; diff --git a/mysql-test/suite/sys_vars/r/binlog_row_metadata_basic.result b/mysql-test/suite/sys_vars/r/binlog_row_metadata_basic.result new file mode 100644 index 00000000000..75971f2d2c6 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/binlog_row_metadata_basic.result @@ -0,0 +1,85 @@ +SELECT @@GLOBAL.binlog_row_metadata; +@@GLOBAL.binlog_row_metadata +MINIMAL +MINIMAL Expected +SELECT @@SESSION.binlog_row_metadata; +ERROR HY000: Variable 'binlog_row_metadata' is a GLOBAL variable +'#---------------------BS_STVARS_002_01----------------------#' +SET @start_value= @@global.binlog_row_metadata; +SELECT COUNT(@@GLOBAL.binlog_row_metadata); +COUNT(@@GLOBAL.binlog_row_metadata) +1 +1 Expected +'#---------------------BS_STVARS_002_02----------------------#' +SET @@GLOBAL.binlog_row_metadata=0; +SELECT @@GLOBAL.binlog_row_metadata; +@@GLOBAL.binlog_row_metadata +MINIMAL +MINIMAL Expected +SET @@GLOBAL.binlog_row_metadata=1; +SELECT @@GLOBAL.binlog_row_metadata; +@@GLOBAL.binlog_row_metadata +FULL +FULL Expected +SET @@GLOBAL.binlog_row_metadata=MINIMAL; +SELECT @@GLOBAL.binlog_row_metadata; +@@GLOBAL.binlog_row_metadata +MINIMAL +MINIMAL Expected +SET @@GLOBAL.binlog_row_metadata=FULL; +SELECT @@GLOBAL.binlog_row_metadata; +@@GLOBAL.binlog_row_metadata +FULL +FULL Expected +SET @@GLOBAL.binlog_row_metadata='MINIMAL'; +SELECT @@GLOBAL.binlog_row_metadata; +@@GLOBAL.binlog_row_metadata +MINIMAL +MINIMAL Expected +SET @@GLOBAL.binlog_row_metadata='FULL'; +SELECT @@GLOBAL.binlog_row_metadata; +@@GLOBAL.binlog_row_metadata +FULL +FULL Expected +'#---------------------BS_STVARS_002_03----------------------#' +SELECT @@GLOBAL.binlog_row_metadata = VARIABLE_VALUE +FROM performance_schema.global_variables +WHERE VARIABLE_NAME='binlog_row_metadata'; +@@GLOBAL.binlog_row_metadata = VARIABLE_VALUE +1 +1 Expected +SELECT COUNT(@@GLOBAL.binlog_row_metadata); +COUNT(@@GLOBAL.binlog_row_metadata) +1 +1 Expected +SELECT COUNT(VARIABLE_VALUE) +FROM performance_schema.global_variables +WHERE VARIABLE_NAME='binlog_row_metadata'; +COUNT(VARIABLE_VALUE) +1 +1 Expected +'#---------------------BS_STVARS_002_04----------------------#' +SELECT @@GLOBAL.binlog_row_metadata = VARIABLE_VALUE +FROM performance_schema.session_variables +WHERE VARIABLE_NAME LIKE 'binlog_row_metadata'; +@@GLOBAL.binlog_row_metadata = VARIABLE_VALUE +1 +'#---------------------BS_STVARS_002_05----------------------#' +SELECT COUNT(@@binlog_row_metadata); +COUNT(@@binlog_row_metadata) +1 +1 Expected +SELECT COUNT(@@GLOBAL.binlog_row_metadata); +COUNT(@@GLOBAL.binlog_row_metadata) +1 +1 Expected +'#---------------------BS_STVARS_002_06----------------------#' +SET GLOBAL binlog_row_metadata = full1; +ERROR 42000: Variable 'binlog_row_metadata' can't be set to the value of 'full1' +SET GLOBAL binlog_row_metadata = "full1"; +ERROR 42000: Variable 'binlog_row_metadata' can't be set to the value of 'full1' +SET GLOBAL binlog_row_metadata = 2; +ERROR 42000: Variable 'binlog_row_metadata' can't be set to the value of '2' +SET GLOBAL binlog_row_metadata = -1; +ERROR 42000: Variable 'binlog_row_metadata' can't be set to the value of '-1' +SET @@global.binlog_row_metadata= @start_value; diff --git a/mysql-test/suite/sys_vars/t/binlog_row_metadata_basic.test b/mysql-test/suite/sys_vars/t/binlog_row_metadata_basic.test new file mode 100644 index 00000000000..e66e8d4d31e --- /dev/null +++ b/mysql-test/suite/sys_vars/t/binlog_row_metadata_basic.test @@ -0,0 +1,124 @@ +################## mysql-test\t\binlog_row_metadata_basic.test ################ +# # +# Variable Name: binlog_row_metadata # +# Scope: Global # +# Data Type: enumeration # +# # +# Creation Date: 2017-01-23 # +# Author : Libing Song # +# # +# # +# Description:Test Cases of Dynamic System Variable binlog_row_metadata # +# that checks the behavior of this variable in the following ways # +# * Value Check # +# * Scope Check # +# # +# Reference: # +# http://dev.mysql.com/doc/refman/8.X/en/server-system-variables.html # +# # +############################################################################### + + +SELECT @@GLOBAL.binlog_row_metadata; +--echo MINIMAL Expected +--error 1238 +SELECT @@SESSION.binlog_row_metadata; + +--echo '#---------------------BS_STVARS_002_01----------------------#' +#################################################################### +# Displaying default value # +#################################################################### +SET @start_value= @@global.binlog_row_metadata; + +SELECT COUNT(@@GLOBAL.binlog_row_metadata); +--echo 1 Expected + +--echo '#---------------------BS_STVARS_002_02----------------------#' +#################################################################### +# Check if Value can set # +#################################################################### +SET @@GLOBAL.binlog_row_metadata=0; +SELECT @@GLOBAL.binlog_row_metadata; +--echo MINIMAL Expected + +SET @@GLOBAL.binlog_row_metadata=1; +SELECT @@GLOBAL.binlog_row_metadata; +--echo FULL Expected + +SET @@GLOBAL.binlog_row_metadata=MINIMAL; +SELECT @@GLOBAL.binlog_row_metadata; +--echo MINIMAL Expected + +SET @@GLOBAL.binlog_row_metadata=FULL; +SELECT @@GLOBAL.binlog_row_metadata; +--echo FULL Expected + +SET @@GLOBAL.binlog_row_metadata='MINIMAL'; +SELECT @@GLOBAL.binlog_row_metadata; +--echo MINIMAL Expected + +SET @@GLOBAL.binlog_row_metadata='FULL'; +SELECT @@GLOBAL.binlog_row_metadata; +--echo FULL Expected + +--echo '#---------------------BS_STVARS_002_03----------------------#' +################################################################# +# Check if the value in GLOBAL Table matches value in variable # +################################################################# + +--disable_warnings +SELECT @@GLOBAL.binlog_row_metadata = VARIABLE_VALUE +FROM performance_schema.global_variables +WHERE VARIABLE_NAME='binlog_row_metadata'; +--enable_warnings +--echo 1 Expected + +SELECT COUNT(@@GLOBAL.binlog_row_metadata); +--echo 1 Expected + +--disable_warnings +SELECT COUNT(VARIABLE_VALUE) +FROM performance_schema.global_variables +WHERE VARIABLE_NAME='binlog_row_metadata'; +--enable_warnings +--echo 1 Expected + + +--echo '#---------------------BS_STVARS_002_04----------------------#' +################################################################# +# Check if the value in SESSION Table matches value in variable # +################################################################# + +SELECT @@GLOBAL.binlog_row_metadata = VARIABLE_VALUE +FROM performance_schema.session_variables +WHERE VARIABLE_NAME LIKE 'binlog_row_metadata'; + +--echo '#---------------------BS_STVARS_002_05----------------------#' +################################################################################ +# Check if binlog_row_metadata can be accessed with and without @@ sign # +################################################################################ + +SELECT COUNT(@@binlog_row_metadata); +--echo 1 Expected +SELECT COUNT(@@GLOBAL.binlog_row_metadata); +--echo 1 Expected + +--echo '#---------------------BS_STVARS_002_06----------------------#' +################################################################################ +# Check if binlog_row_metadata can handle invalid values correctly # +################################################################################ +--error 1231 # Cannot set the value +SET GLOBAL binlog_row_metadata = full1; + +--error 1231 +SET GLOBAL binlog_row_metadata = "full1"; + +--error 1231 +SET GLOBAL binlog_row_metadata = 2; + +--error 1231 +SET GLOBAL binlog_row_metadata = -1; + +SET @@global.binlog_row_metadata= @start_value; + + diff --git a/sql/log_event.cc b/sql/log_event.cc index 351caa96446..6e5aa49f837 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -3428,7 +3428,8 @@ Table_map_log_event::Table_map_log_event(const char *buf, uint event_len, m_colcnt(0), m_coltype(0), m_memory(NULL), m_table_id(ULONGLONG_MAX), m_flags(0), m_data_size(0), m_field_metadata(0), m_field_metadata_size(0), - m_null_bits(0), m_meta_memory(NULL) + m_null_bits(0), m_meta_memory(NULL), + m_optional_metadata_len(0), m_optional_metadata(NULL) { unsigned int bytes_read= 0; DBUG_ENTER("Table_map_log_event::Table_map_log_event(const char*,uint,...)"); @@ -3517,6 +3518,18 @@ Table_map_log_event::Table_map_log_event(const char *buf, uint event_len, memcpy(m_field_metadata, ptr_after_colcnt, m_field_metadata_size); ptr_after_colcnt= (uchar*)ptr_after_colcnt + m_field_metadata_size; memcpy(m_null_bits, ptr_after_colcnt, num_null_bytes); + ptr_after_colcnt= (unsigned char*)ptr_after_colcnt + num_null_bytes; + } + + bytes_read= (uint) (ptr_after_colcnt - (uchar *)buf); + + /* After null_bits field, there are some new fields for extra metadata. */ + if (bytes_read < event_len) + { + m_optional_metadata_len= event_len - bytes_read; + m_optional_metadata= + static_cast<unsigned char*>(my_malloc(m_optional_metadata_len, MYF(MY_WME))); + memcpy(m_optional_metadata, ptr_after_colcnt, m_optional_metadata_len); } } @@ -3528,8 +3541,328 @@ Table_map_log_event::~Table_map_log_event() { my_free(m_meta_memory); my_free(m_memory); + my_free(m_optional_metadata); + m_optional_metadata= NULL; } +/** + Parses SIGNEDNESS field. + + @param[out] vec stores the signedness flags extracted from field. + @param[in] field SIGNEDNESS field in table_map_event. + @param[in] length length of the field + */ +static void parse_signedness(std::vector<bool> &vec, + unsigned char *field, unsigned int length) +{ + for (unsigned int i= 0; i < length; i++) + { + for (unsigned char c= 0x80; c != 0; c>>= 1) + vec.push_back(field[i] & c); + } + } + +/** + Parses DEFAULT_CHARSET field. + + @param[out] default_charset stores collation numbers extracted from field. + @param[in] field DEFAULT_CHARSET field in table_map_event. + @param[in] length length of the field + */ +static void parse_default_charset(Table_map_log_event::Optional_metadata_fields:: + Default_charset &default_charset, + unsigned char *field, unsigned int length) +{ + unsigned char* p= field; + + default_charset.default_charset= net_field_length(&p); + while (p < field + length) + { + unsigned int col_index= net_field_length(&p); + unsigned int col_charset= net_field_length(&p); + + default_charset.charset_pairs.push_back(std::make_pair(col_index, + col_charset)); + } +} + +/** + Parses COLUMN_CHARSET field. + + @param[out] vec stores collation numbers extracted from field. + @param[in] field COLUMN_CHARSET field in table_map_event. + @param[in] length length of the field + */ +static void parse_column_charset(std::vector<unsigned int> &vec, + unsigned char *field, unsigned int length) +{ + unsigned char* p= field; + + while (p < field + length) + vec.push_back(net_field_length(&p)); +} + +/** + Parses COLUMN_NAME field. + + @param[out] vec stores column names extracted from field. + @param[in] field COLUMN_NAME field in table_map_event. + @param[in] length length of the field + */ +static void parse_column_name(std::vector<std::string> &vec, + unsigned char *field, unsigned int length) +{ + unsigned char* p= field; + + while (p < field + length) + { + unsigned len= net_field_length(&p); + vec.push_back(std::string(reinterpret_cast<char *>(p), len)); + p+= len; + } +} + +/** + Parses SET_STR_VALUE/ENUM_STR_VALUE field. + + @param[out] vec stores SET/ENUM column's string values extracted from + field. Each SET/ENUM column's string values are stored + into a string separate vector. All of them are stored + in 'vec'. + @param[in] field COLUMN_NAME field in table_map_event. + @param[in] length length of the field + */ +static void parse_set_str_value(std::vector<Table_map_log_event:: + Optional_metadata_fields::str_vector> &vec, + unsigned char *field, unsigned int length) +{ + unsigned char* p= field; + + while (p < field + length) + { + unsigned int count= net_field_length(&p); + + vec.push_back(std::vector<std::string>()); + for (unsigned int i= 0; i < count; i++) + { + unsigned len1= net_field_length(&p); + vec.back().push_back(std::string(reinterpret_cast<char *>(p), len1)); + p+= len1; + } + } +} + +/** + Parses GEOMETRY_TYPE field. + + @param[out] vec stores geometry column's types extracted from field. + @param[in] field GEOMETRY_TYPE field in table_map_event. + @param[in] length length of the field + */ +static void parse_geometry_type(std::vector<unsigned int> &vec, + unsigned char *field, unsigned int length) +{ + unsigned char* p= field; + + while (p < field + length) + vec.push_back(net_field_length(&p)); +} + +/** + Parses SIMPLE_PRIMARY_KEY field. + + @param[out] vec stores primary key's column information extracted from + field. Each column has an index and a prefix which are + stored as a unit_pair. prefix is always 0 for + SIMPLE_PRIMARY_KEY field. + @param[in] field SIMPLE_PRIMARY_KEY field in table_map_event. + @param[in] length length of the field + */ +static void parse_simple_pk(std::vector<Table_map_log_event:: + Optional_metadata_fields::uint_pair> &vec, + unsigned char *field, unsigned int length) +{ + unsigned char* p= field; + + while (p < field + length) + vec.push_back(std::make_pair(net_field_length(&p), 0)); +} + +/** + Parses PRIMARY_KEY_WITH_PREFIX field. + + @param[out] vec stores primary key's column information extracted from + field. Each column has an index and a prefix which are + stored as a unit_pair. + @param[in] field PRIMARY_KEY_WITH_PREFIX field in table_map_event. + @param[in] length length of the field + */ + +static void parse_pk_with_prefix(std::vector<Table_map_log_event:: + Optional_metadata_fields::uint_pair> &vec, + unsigned char *field, unsigned int length) +{ + unsigned char* p= field; + + while (p < field + length) + { + unsigned int col_index= net_field_length(&p); + unsigned int col_prefix= net_field_length(&p); + vec.push_back(std::make_pair(col_index, col_prefix)); + } +} + +Table_map_log_event::Optional_metadata_fields:: +Optional_metadata_fields(unsigned char* optional_metadata, + unsigned int optional_metadata_len) +{ + unsigned char* field= optional_metadata; + + if (optional_metadata == NULL) + return; + + while (field < optional_metadata + optional_metadata_len) + { + unsigned int len; + Optional_metadata_field_type type= + static_cast<Optional_metadata_field_type>(field[0]); + + // Get length and move field to the value. + field++; + len= net_field_length(&field); + + switch(type) + { + case SIGNEDNESS: + parse_signedness(m_signedness, field, len); + break; + case DEFAULT_CHARSET: + parse_default_charset(m_default_charset, field, len); + break; + case COLUMN_CHARSET: + parse_column_charset(m_column_charset, field, len); + break; + case COLUMN_NAME: + parse_column_name(m_column_name, field, len); + break; + case SET_STR_VALUE: + parse_set_str_value(m_set_str_value, field, len); + break; + case ENUM_STR_VALUE: + parse_set_str_value(m_enum_str_value, field, len); + break; + case GEOMETRY_TYPE: + parse_geometry_type(m_geometry_type, field, len); + break; + case SIMPLE_PRIMARY_KEY: + parse_simple_pk(m_primary_key, field, len); + break; + case PRIMARY_KEY_WITH_PREFIX: + parse_pk_with_prefix(m_primary_key, field, len); + break; + case ENUM_AND_SET_DEFAULT_CHARSET: + parse_default_charset(m_enum_and_set_default_charset, field, len); + break; + case ENUM_AND_SET_COLUMN_CHARSET: + parse_column_charset(m_enum_and_set_column_charset, field, len); + break; + default: + DBUG_ASSERT(0); + } + // next field + field+= len; + } +} +#ifndef MYSQL_SERVER +/** + Interface for iterator over charset columns. +*/ +class Table_map_log_event::Charset_iterator +{ + public: + typedef Table_map_log_event::Optional_metadata_fields::Default_charset + Default_charset; + virtual const CHARSET_INFO *next()= 0; + virtual ~Charset_iterator(){}; + /** + Factory method to create an instance of the appropriate subclass. + */ + static std::unique_ptr<Charset_iterator> create_charset_iterator( + const Default_charset &default_charset, + const std::vector<uint> &column_charset); +}; + +/** + Implementation of charset iterator for the DEFAULT_CHARSET type. +*/ +class Table_map_log_event::Default_charset_iterator : public Charset_iterator +{ + public: + Default_charset_iterator(const Default_charset &default_charset) + : m_iterator(default_charset.charset_pairs.begin()), + m_end(default_charset.charset_pairs.end()), + m_column_index(0), + m_default_charset_info( + get_charset(default_charset.default_charset, 0)) {} + + const CHARSET_INFO *next() override { + const CHARSET_INFO *ret; + if (m_iterator != m_end && m_iterator->first == m_column_index) { + ret = get_charset(m_iterator->second, 0); + m_iterator++; + } else + ret = m_default_charset_info; + m_column_index++; + return ret; + } + ~Default_charset_iterator(){}; + + private: + std::vector<Optional_metadata_fields::uint_pair>::const_iterator m_iterator, + m_end; + uint m_column_index; + const CHARSET_INFO *m_default_charset_info; +}; +//Table_map_log_event::Default_charset_iterator::~Default_charset_iterator(){int a=8;a++; a--;}; +/** + Implementation of charset iterator for the COLUMNT_CHARSET type. +*/ +class Table_map_log_event::Column_charset_iterator : public Charset_iterator +{ + public: + Column_charset_iterator(const std::vector<uint> &column_charset) + : m_iterator(column_charset.begin()), m_end(column_charset.end()) {} + + const CHARSET_INFO *next() override { + const CHARSET_INFO *ret = nullptr; + if (m_iterator != m_end) { + ret = get_charset(*m_iterator, 0); + m_iterator++; + } + return ret; + } + + ~Column_charset_iterator(){}; + private: + std::vector<uint>::const_iterator m_iterator; + std::vector<uint>::const_iterator m_end; +}; +//Table_map_log_event::Column_charset_iterator::~Column_charset_iterator(){int a=8;a++; a--;}; + +std::unique_ptr<Table_map_log_event::Charset_iterator> +Table_map_log_event::Charset_iterator::create_charset_iterator( + const Default_charset &default_charset, + const std::vector<uint> &column_charset) +{ + if (!default_charset.empty()) + return std::unique_ptr<Charset_iterator>( + new Default_charset_iterator(default_charset)); + else + return std::unique_ptr<Charset_iterator>( + new Column_charset_iterator(column_charset)); +} +#endif + /************************************************************************** Write_rows_log_event member functions diff --git a/sql/log_event.h b/sql/log_event.h index 274182af841..dcfcf8a5622 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -35,6 +35,10 @@ #include <my_bitmap.h> #include "rpl_constants.h" +#include <vector> +#include <string> +#include <functional> +#include <memory> #ifdef MYSQL_CLIENT #include "sql_const.h" @@ -791,6 +795,43 @@ enum Int_event_type INVALID_INT_EVENT = 0, LAST_INSERT_ID_EVENT = 1, INSERT_ID_EVENT = 2 }; +static inline bool is_numeric_type(uint type) +{ + switch (type) + { + case MYSQL_TYPE_TINY: + case MYSQL_TYPE_SHORT: + case MYSQL_TYPE_INT24: + case MYSQL_TYPE_LONG: + case MYSQL_TYPE_LONGLONG: + case MYSQL_TYPE_NEWDECIMAL: + case MYSQL_TYPE_FLOAT: + case MYSQL_TYPE_DOUBLE: + return true; + default: + return false; + } + return false; +} + +static inline bool is_character_type(uint type) +{ + switch (type) + { + case MYSQL_TYPE_STRING: + case MYSQL_TYPE_VAR_STRING: + case MYSQL_TYPE_VARCHAR: + case MYSQL_TYPE_BLOB: + return true; + default: + return false; + } +} + +static inline bool is_enum_or_set_type(uint type) { + return type == MYSQL_TYPE_ENUM || type == MYSQL_TYPE_SET; +} + #ifdef MYSQL_SERVER class String; @@ -881,6 +922,7 @@ typedef struct st_print_event_info statement for it. */ bool skip_replication; + bool print_table_metadata; /* These two caches are used by the row-based replication events to @@ -4063,6 +4105,18 @@ class Annotate_rows_log_event: public Log_event ninth is in the least significant bit of the second byte, and so on. </td> </tr> + <tr> + <td>optional metadata fields</td> + <td>optional metadata fields are stored in Type, Length, Value(TLV) format. + Type takes 1 byte. Length is a packed integer value. Values takes + Length bytes. + </td> + <td>There are some optional metadata defined. They are listed in the table + @ref Table_table_map_event_optional_metadata. Optional metadata fields + follow null_bits. Whether binlogging an optional metadata is decided by the + server. The order is not defined, so they can be binlogged in any order. + </td> + </tr> </table> @@ -4268,6 +4322,123 @@ class Annotate_rows_log_event: public Log_event </tr> </table> + The table below lists all optional metadata types, along with the numerical + identifier for it and the size and interpretation of meta-data used + to describe the type. + + @anchor Table_table_map_event_optional_metadata + <table> + <caption>Table_map_event optional metadata types: numerical identifier and + metadata. Optional metadata fields are stored in TLV fields. + Format of values are described in this table. </caption> + <tr> + <th>Type</th> + <th>Description</th> + <th>Format</th> + </tr> + <tr> + <td>SIGNEDNESS</td> + <td>signedness of numeric colums. This is included for all values of + binlog_row_metadata.</td> + <td>For each numeric column, a bit indicates whether the numeric + colunm has unsigned flag. 1 means it is unsigned. The number of + bytes needed for this is int((column_count + 7) / 8). The order is + the same as the order of column_type field.</td> + </tr> + <tr> + <td>DEFAULT_CHARSET</td> + <td>Charsets of character columns. It has a default charset for + the case that most of character columns have same charset and the + most used charset is binlogged as default charset.Collation + numbers are binlogged for identifying charsets. They are stored in + packed length format. Either DEFAULT_CHARSET or COLUMN_CHARSET is + included for all values of binlog_row_metadata.</td> + <td>Default charset's collation is logged first. The charsets which are not + same to default charset are logged following default charset. They are + logged as column index and charset collation number pair sequence. The + column index is counted only in all character columns. The order is same to + the order of column_type + field. </td> + </tr> + <tr> + <td>COLUMN_CHARSET</td> + <td>Charsets of character columns. For the case that most of columns have + different charsets, this field is logged. It is never logged with + DEFAULT_CHARSET together. Either DEFAULT_CHARSET or COLUMN_CHARSET is + included for all values of binlog_row_metadata.</td> + <td>It is a collation number sequence for all character columns.</td> + </tr> + <tr> + <td>COLUMN_NAME</td> + <td>Names of columns. This is only included if + binlog_row_metadata=FULL.</td> + <td>A sequence of column names. For each column name, 1 byte for + the string length in bytes is followed by a string without null + terminator.</td> + </tr> + <tr> + <td>SET_STR_VALUE</td> + <td>The string values of SET columns. This is only included if + binlog_row_metadata=FULL.</td> + <td>For each SET column, a pack_length representing the value + count is followed by a sequence of length and string pairs. length + is the byte count in pack_length format. The string has no null + terminator.</td> + </tr> + <tr> + <td>ENUM_STR_VALUE</td> + <td>The string values is ENUM columns. This is only included + if binlog_row_metadata=FULL.</td> + <td>The format is the same as SET_STR_VALUE.</td> + </tr> + <tr> + <td>GEOMETRY_TYPE</td> + <td>The real type of geometry columns. This is only included + if binlog_row_metadata=FULL.</td> + <td>A sequence of real type of geometry columns are stored in pack_length + format. </td> + </tr> + <tr> + <td>SIMPLE_PRIMARY_KEY</td> + <td>The primary key without any prefix. This is only included + if binlog_row_metadata=FULL and there is a primary key where every + key part covers an entire column.</td> + <td>A sequence of column indexes. The indexes are stored in pack_length + format.</td> + </tr> + <tr> + <td>PRIMARY_KEY_WITH_PREFIX</td> + <td>The primary key with some prefix. It doesn't appear together with + SIMPLE_PRIMARY_KEY. This is only included if + binlog_row_metadata=FULL and there is a primary key where some key + part covers a prefix of the column.</td> + <td>A sequence of column index and prefix length pairs. Both + column index and prefix length are in pack_length format. Prefix length + 0 means that the whole column value is used.</td> + </tr> + <tr> + <td>ENUM_AND_SET_DEFAULT_CHARSET</td> + <td>Charsets of ENUM and SET columns. It has the same layout as + DEFAULT_CHARSET. If there are SET or ENUM columns and + binlog_row_metadata=FULL, exactly one of + ENUM_AND_SET_DEFAULT_CHARSET and ENUM_AND_SET_COLUMN_CHARSET + appears (the encoder chooses the representation that uses the + least amount of space). Otherwise, none of them appears.</td> + <td>The same format as for DEFAULT_CHARSET, except it counts ENUM + and SET columns rather than character columns.</td> + </tr> + <tr> + <td>ENUM_AND_SET_COLUMN_CHARSET</td> + <td>Charsets of ENUM and SET columns. It has the same layout as + COLUMN_CHARSET. If there are SET or ENUM columns and + binlog_row_metadata=FULL, exactly one of + ENUM_AND_SET_DEFAULT_CHARSET and ENUM_AND_SET_COLUMN_CHARSET + appears (the encoder chooses the representation that uses the + least amount of space). Otherwise, none of them appears.</td> + <td>The same format as for COLUMN_CHARSET, except it counts ENUM + and SET columns rather than character columns.</td> + </tr> + </table> */ class Table_map_log_event : public Log_event { @@ -4303,6 +4474,117 @@ class Table_map_log_event : public Log_event }; typedef uint16 flag_set; + /** + DEFAULT_CHARSET and COLUMN_CHARSET don't appear together, and + ENUM_AND_SET_DEFAULT_CHARSET and ENUM_AND_SET_COLUMN_CHARSET don't + appear together. They are just alternative ways to pack character + set information. When binlogging, it logs character sets in the + way that occupies least storage. + + SIMPLE_PRIMARY_KEY and PRIMARY_KEY_WITH_PREFIX don't appear together. + SIMPLE_PRIMARY_KEY is for the primary keys which only use whole values of + pk columns. PRIMARY_KEY_WITH_PREFIX is + for the primary keys which just use part value of pk columns. + */ + enum Optional_metadata_field_type + { + SIGNEDNESS = 1, // UNSIGNED flag of numeric columns + DEFAULT_CHARSET, /* Character set of string columns, optimized to + minimize space when many columns have the + same charset. */ + COLUMN_CHARSET, /* Character set of string columns, optimized to + minimize space when columns have many + different charsets. */ + COLUMN_NAME, + SET_STR_VALUE, // String value of SET columns + ENUM_STR_VALUE, // String value of ENUM columns + GEOMETRY_TYPE, // Real type of geometry columns + SIMPLE_PRIMARY_KEY, // Primary key without prefix + PRIMARY_KEY_WITH_PREFIX, // Primary key with prefix + ENUM_AND_SET_DEFAULT_CHARSET, /* Character set of enum and set + columns, optimized to minimize + space when many columns have the + same charset. */ + ENUM_AND_SET_COLUMN_CHARSET, /* Character set of enum and set + columns, optimized to minimize + space when many columns have the + same charset. */ + }; + /** + Metadata_fields organizes m_optional_metadata into a structured format which + is easy to access. + */ + struct Optional_metadata_fields + { + typedef std::pair<unsigned int, unsigned int> uint_pair; + typedef std::vector<std::string> str_vector; + + struct Default_charset + { + Default_charset() : default_charset(0) {} + bool empty() const { return default_charset == 0; } + + // Default charset for the columns which are not in charset_pairs. + unsigned int default_charset; + + /* The uint_pair means <column index, column charset number>. */ + std::vector<uint_pair> charset_pairs; + }; + + // Contents of DEFAULT_CHARSET field is converted into Default_charset. + Default_charset m_default_charset; + // Contents of ENUM_AND_SET_DEFAULT_CHARSET are converted into + // Default_charset. + Default_charset m_enum_and_set_default_charset; + std::vector<bool> m_signedness; + // Character set number of every string column + std::vector<unsigned int> m_column_charset; + // Character set number of every ENUM or SET column. + std::vector<unsigned int> m_enum_and_set_column_charset; + std::vector<std::string> m_column_name; + // each str_vector stores values of one enum/set column + std::vector<str_vector> m_enum_str_value; + std::vector<str_vector> m_set_str_value; + std::vector<unsigned int> m_geometry_type; + /* + The uint_pair means <column index, prefix length>. Prefix length is 0 if + whole column value is used. + */ + std::vector<uint_pair> m_primary_key; + + /* + It parses m_optional_metadata and populates into above variables. + + @param[in] optional_metadata points to the begin of optional metadata + fields in table_map_event. + @param[in] optional_metadata_len length of optional_metadata field. + */ + Optional_metadata_fields(unsigned char* optional_metadata, + unsigned int optional_metadata_len); + }; + + /** + Print column metadata. Its format looks like: + # Columns(colume_name type, colume_name type, ...) + if colume_name field is not logged into table_map_log_event, then + only type is printed. + + @@param[out] file the place where colume metadata is printed + @@param[in] The metadata extracted from optional metadata fields + */ + void print_columns(IO_CACHE *file, + const Optional_metadata_fields &fields); + /** + Print primary information. Its format looks like: + # Primary Key(colume_name, column_name(prifix), ...) + if colume_name field is not logged into table_map_log_event, then + colume index is printed. + + @@param[out] file the place where primary key is printed + @@param[in] The metadata extracted from optional metadata fields + */ + void print_primary_key(IO_CACHE *file, + const Optional_metadata_fields &fields); /* Special constants representing sets of flags */ enum @@ -4369,6 +4651,49 @@ class Table_map_log_event : public Log_event #ifdef MYSQL_SERVER TABLE *m_table; + + // Metadata fields buffer + StringBuffer<1024> m_metadata_buf; + + /** + Capture the optional metadata fields which should be logged into + table_map_log_event and serialize them into m_metadata_buf. + */ + void init_metadata_fields(); + bool init_signedness_field(); + /** + Capture and serialize character sets. Character sets for + character columns (TEXT etc) and character sets for ENUM and SET + columns are stored in different metadata fields. The reason is + that TEXT character sets are included even when + binlog_row_metadata=MINIMAL, whereas ENUM and SET character sets + are included only when binlog_row_metadata=FULL. + + @param include_type Predicate to determine if a given Field object + is to be included in the metadata field. + + @param default_charset_type Type code when storing in "default + charset" format. (See comment above Table_maps_log_event in + libbinlogevents/include/rows_event.h) + + @param column_charset_type Type code when storing in "column + charset" format. (See comment above Table_maps_log_event in + libbinlogevents/include/rows_event.h) + */ + bool init_charset_field(std::function<bool(const Field *)> include_type, + Optional_metadata_field_type default_charset_type, + Optional_metadata_field_type column_charset_type); + bool init_column_name_field(); + bool init_set_str_value_field(); + bool init_enum_str_value_field(); + bool init_geometry_type_field(); + bool init_primary_key_field(); +#endif + +#ifndef MYSQL_SERVER + class Charset_iterator; + class Default_charset_iterator; + class Column_charset_iterator; #endif char const *m_dbnam; size_t m_dblen; @@ -4390,6 +4715,8 @@ class Table_map_log_event : public Log_event ulong m_field_metadata_size; uchar *m_null_bits; uchar *m_meta_memory; + unsigned int m_optional_metadata_len; + unsigned char *m_optional_metadata; }; diff --git a/sql/log_event_client.cc b/sql/log_event_client.cc index 57b217813e2..86c96f8274c 100644 --- a/sql/log_event_client.cc +++ b/sql/log_event_client.cc @@ -26,10 +26,11 @@ #endif -static bool pretty_print_str(IO_CACHE* cache, const char* str, int len) +static bool pretty_print_str(IO_CACHE* cache, const char* str, + size_t len, bool identifier) { const char* end = str + len; - if (my_b_write_byte(cache, '\'')) + if (my_b_write_byte(cache, identifier ? '`' : '\'')) goto err; while (str < end) @@ -52,12 +53,39 @@ static bool pretty_print_str(IO_CACHE* cache, const char* str, int len) if (unlikely(error)) goto err; } - return my_b_write_byte(cache, '\''); + return my_b_write_byte(cache, identifier ? '`' : '\''); err: return 1; } +/** + Print src as an string enclosed with "'" + + @param[out] cache IO_CACHE where the string will be printed. + @param[in] str the string will be printed. + @param[in] len length of the string. +*/ +static inline bool pretty_print_str(IO_CACHE* cache, const char* str, + size_t len) +{ + return pretty_print_str(cache, str, len, false); +} + +/** + Print src as an identifier enclosed with "`" + + @param[out] cache IO_CACHE where the identifier will be printed. + @param[in] str the string will be printed. + @param[in] len length of the string. + */ +static inline bool pretty_print_identifier(IO_CACHE* cache, const char* str, + size_t len) +{ + return pretty_print_str(cache, str, len, true); +} + + /** Prints a "session_var=value" string. Used by mysqlbinlog to print some SET @@ -3110,6 +3138,15 @@ bool Table_map_log_event::print(FILE *file, PRINT_EVENT_INFO *print_event_info) } if (!print_event_info->short_form || print_event_info->print_row_count) { + + if (print_event_info->print_table_metadata) + { + Optional_metadata_fields fields(m_optional_metadata, + m_optional_metadata_len); + + print_columns(&print_event_info->head_cache, fields); + print_primary_key(&print_event_info->head_cache, fields); + } bool do_print_encoded= print_event_info->base64_output_mode != BASE64_OUTPUT_NEVER && print_event_info->base64_output_mode != BASE64_OUTPUT_DECODE_ROWS && @@ -3127,6 +3164,318 @@ bool Table_map_log_event::print(FILE *file, PRINT_EVENT_INFO *print_event_info) return 1; } +/** + return the string name of a type. + + @param[in] type type of a column + @param[in|out] meta_ptr the meta_ptr of the column. If the type doesn't have + metadata, it will not change meta_ptr, otherwise + meta_ptr will be moved to the end of the column's + metadat. + @param[in] cs charset of the column if it is a character column. + @param[out] typestr buffer to storing the string name of the type + @param[in] typestr_length length of typestr + @param[in] geometry_type internal geometry_type + */ +static void get_type_name(uint type, unsigned char** meta_ptr, + const CHARSET_INFO *cs, char *typestr, + uint typestr_length, unsigned int geometry_type) +{ + switch (type) { + case MYSQL_TYPE_LONG: + my_snprintf(typestr, typestr_length, "%s", "INT"); + break; + case MYSQL_TYPE_TINY: + my_snprintf(typestr, typestr_length, "TINYINT"); + break; + case MYSQL_TYPE_SHORT: + my_snprintf(typestr, typestr_length, "SMALLINT"); + break; + case MYSQL_TYPE_INT24: + my_snprintf(typestr, typestr_length, "MEDIUMINT"); + break; + case MYSQL_TYPE_LONGLONG: + my_snprintf(typestr, typestr_length, "BIGINT"); + break; + case MYSQL_TYPE_NEWDECIMAL: + my_snprintf(typestr, typestr_length, "DECIMAL(%d,%d)", + (*meta_ptr)[0], (*meta_ptr)[1]); + (*meta_ptr)+= 2; + break; + case MYSQL_TYPE_FLOAT: + my_snprintf(typestr, typestr_length, "FLOAT"); + (*meta_ptr)++; + break; + case MYSQL_TYPE_DOUBLE: + my_snprintf(typestr, typestr_length, "DOUBLE"); + (*meta_ptr)++; + break; + case MYSQL_TYPE_BIT: + my_snprintf(typestr, typestr_length, "BIT(%d)", + (((*meta_ptr)[0])) + (*meta_ptr)[1]*8); + (*meta_ptr)+= 2; + break; + case MYSQL_TYPE_TIMESTAMP2: + if (**meta_ptr != 0) + my_snprintf(typestr, typestr_length, "TIMESTAMP(%d)", **meta_ptr); + else + my_snprintf(typestr, typestr_length, "TIMESTAMP"); + (*meta_ptr)++; + break; + case MYSQL_TYPE_DATETIME2: + if (**meta_ptr != 0) + my_snprintf(typestr, typestr_length, "DATETIME(%d)", **meta_ptr); + else + my_snprintf(typestr, typestr_length, "DATETIME"); + (*meta_ptr)++; + break; + case MYSQL_TYPE_TIME2: + if (**meta_ptr != 0) + my_snprintf(typestr, typestr_length, "TIME(%d)", **meta_ptr); + else + my_snprintf(typestr, typestr_length, "TIME"); + (*meta_ptr)++; + break; + case MYSQL_TYPE_NEWDATE: + case MYSQL_TYPE_DATE: + my_snprintf(typestr, typestr_length, "DATE"); + break; + case MYSQL_TYPE_YEAR: + my_snprintf(typestr, typestr_length, "YEAR"); + break; + case MYSQL_TYPE_ENUM: + my_snprintf(typestr, typestr_length, "ENUM"); + (*meta_ptr)+= 2; + break; + case MYSQL_TYPE_SET: + my_snprintf(typestr, typestr_length, "SET"); + (*meta_ptr)+= 2; + break; + case MYSQL_TYPE_BLOB: + { + bool is_text= (cs && cs->number != my_charset_bin.number); + const char *names[5][2] = { + {"INVALID_BLOB(%d)", "INVALID_TEXT(%d)"}, + {"TINYBLOB", "TINYTEXT"}, + {"BLOB", "TEXT"}, + {"MEDIUMBLOB", "MEDIUMTEXT"}, + {"LONGBLOB", "LONGTEXT"} + }; + unsigned char size= **meta_ptr; + + if (size == 0 || size > 4) + my_snprintf(typestr, typestr_length, names[0][is_text], size); + else + my_snprintf(typestr, typestr_length, names[**meta_ptr][is_text]); + + (*meta_ptr)++; + } + break; + case MYSQL_TYPE_VARCHAR: + case MYSQL_TYPE_VAR_STRING: + if (cs && cs->number != my_charset_bin.number) + my_snprintf(typestr, typestr_length, "VARCHAR(%d)", + uint2korr(*meta_ptr)/cs->mbmaxlen); + else + my_snprintf(typestr, typestr_length, "VARBINARY(%d)", + uint2korr(*meta_ptr)); + + (*meta_ptr)+= 2; + break; + case MYSQL_TYPE_STRING: + { + uint byte0= (*meta_ptr)[0]; + uint byte1= (*meta_ptr)[1]; + uint len= (((byte0 & 0x30) ^ 0x30) << 4) | byte1; + + if (cs && cs->number != my_charset_bin.number) + my_snprintf(typestr, typestr_length, "CHAR(%d)", len/cs->mbmaxlen); + else + my_snprintf(typestr, typestr_length, "BINARY(%d)", len); + + (*meta_ptr)+= 2; + } + break; + case MYSQL_TYPE_JSON: + my_snprintf(typestr, typestr_length, "JSON"); + (*meta_ptr)++; + break; + case MYSQL_TYPE_GEOMETRY: + { + const char* names[8] = { + "GEOMETRY", "POINT", "LINESTRING", "POLYGON", "MULTIPOINT", + "MULTILINESTRING", "MULTIPOLYGON", "GEOMETRYCOLLECTION" + }; + if (geometry_type < 8) + my_snprintf(typestr, typestr_length, names[geometry_type]); + else + my_snprintf(typestr, typestr_length, "INVALID_GEOMETRY_TYPE(%u)", + geometry_type); + (*meta_ptr)++; + } + break; + default: + *typestr= 0; + break; + } +} + +void Table_map_log_event::print_columns(IO_CACHE *file, + const Optional_metadata_fields &fields) +{ + unsigned char* field_metadata_ptr= m_field_metadata; + std::vector<bool>::const_iterator signedness_it= fields.m_signedness.begin(); + + std::unique_ptr<Charset_iterator> charset_it = + Charset_iterator::create_charset_iterator(fields.m_default_charset, + fields.m_column_charset); + std::unique_ptr<Charset_iterator> enum_and_set_charset_it = + Charset_iterator::create_charset_iterator( + fields.m_enum_and_set_default_charset, + fields.m_enum_and_set_column_charset); + std::vector<std::string>::const_iterator col_names_it= + fields.m_column_name.begin(); + std::vector<Optional_metadata_fields::str_vector>::const_iterator + set_str_values_it= fields.m_set_str_value.begin(); + std::vector<Optional_metadata_fields::str_vector>::const_iterator + enum_str_values_it= fields.m_enum_str_value.begin(); + std::vector<unsigned int>::const_iterator geometry_type_it= + fields.m_geometry_type.begin(); + + uint geometry_type= 0; + + my_b_printf(file, "# Columns("); + + for (unsigned long i= 0; i < m_colcnt; i++) + { + uint real_type = m_coltype[i]; + if (real_type == MYSQL_TYPE_STRING && + (*field_metadata_ptr == MYSQL_TYPE_ENUM || + *field_metadata_ptr == MYSQL_TYPE_SET)) + real_type= *field_metadata_ptr; + + // Get current column's collation id if it is a character, enum, + // or set column + const CHARSET_INFO *cs = NULL; + if (is_character_type(real_type)) + cs = charset_it->next(); + else if (is_enum_or_set_type(real_type)) + cs = enum_and_set_charset_it->next(); + + // Print column name + if (col_names_it != fields.m_column_name.end()) + { + pretty_print_identifier(file, col_names_it->c_str(), col_names_it->size()); + my_b_printf(file, " "); + col_names_it++; + } + + + // update geometry_type for geometry columns + if (real_type == MYSQL_TYPE_GEOMETRY) + { + geometry_type= (geometry_type_it != fields.m_geometry_type.end()) ? + *geometry_type_it++ : 0; + } + + // print column type + const uint TYPE_NAME_LEN = 100; + char type_name[TYPE_NAME_LEN]; + get_type_name(real_type, &field_metadata_ptr, cs, type_name, + TYPE_NAME_LEN, geometry_type); + + if (type_name[0] == '\0') + { + my_b_printf(file, "INVALID_TYPE(%d)", real_type); + continue; + } + my_b_printf(file, "%s", type_name); + + // Print UNSIGNED for numeric column + if (is_numeric_type(real_type) && + signedness_it != fields.m_signedness.end()) + { + if (*signedness_it == true) + my_b_printf(file, " UNSIGNED"); + signedness_it++; + } + + // if the column is not marked as 'null', print 'not null' + if (!(m_null_bits[(i / 8)] & (1 << (i % 8)))) + my_b_printf(file, " NOT NULL"); + + // Print string values of SET and ENUM column + const Optional_metadata_fields::str_vector *str_values= NULL; + if (real_type == MYSQL_TYPE_ENUM && + enum_str_values_it != fields.m_enum_str_value.end()) + { + str_values= &(*enum_str_values_it); + enum_str_values_it++; + } + else if (real_type == MYSQL_TYPE_SET && + set_str_values_it != fields.m_set_str_value.end()) + { + str_values= &(*set_str_values_it); + set_str_values_it++; + } + + if (str_values != NULL) + { + const char *separator= "("; + for (Optional_metadata_fields::str_vector::const_iterator it= + str_values->begin(); it != str_values->end(); it++) + { + my_b_printf(file, "%s", separator); + pretty_print_str(file, it->c_str(), it->size()); + separator= ","; + } + my_b_printf(file, ")"); + } + // Print column character set, except in text columns with binary collation + if (cs != NULL && + (is_enum_or_set_type(real_type) || cs->number != my_charset_bin.number)) + my_b_printf(file, " CHARSET %s COLLATE %s", cs->csname, cs->name); + + // Print column character set, except in text columns with binary collation + if (cs != NULL && + (is_enum_or_set_type(real_type) || cs->number != my_charset_bin.number)) + my_b_printf(file, " CHARSET %s COLLATE %s", cs->csname, cs->name); + if (i != m_colcnt - 1) my_b_printf(file, ",\n# "); + } + my_b_printf(file, ")"); + my_b_printf(file, "\n"); +} + +void Table_map_log_event::print_primary_key + (IO_CACHE *file,const Optional_metadata_fields &fields) +{ + if (!fields.m_primary_key.empty()) + { + my_b_printf(file, "# Primary Key("); + + std::vector<Optional_metadata_fields::uint_pair>::const_iterator it= + fields.m_primary_key.begin(); + + for (; it != fields.m_primary_key.end(); it++) + { + if (it != fields.m_primary_key.begin()) + my_b_printf(file, ", "); + + // Print column name or column index + if (it->first >= fields.m_column_name.size()) + my_b_printf(file, "%u", it->first); + else + my_b_printf(file, "%s", fields.m_column_name[it->first].c_str()); + + // Print prefix length + if (it->second != 0) + my_b_printf(file, "(%u)", it->second); + } + + my_b_printf(file, ")\n"); + } +} + bool Write_rows_log_event::print(FILE *file, PRINT_EVENT_INFO* print_event_info) { diff --git a/sql/log_event_server.cc b/sql/log_event_server.cc index 70b1c1538e9..1dd70ba7fea 100644 --- a/sql/log_event_server.cc +++ b/sql/log_event_server.cc @@ -5915,6 +5915,11 @@ int Table_map_log_event::save_field_metadata() { DBUG_PRINT("debug", ("field_type: %d", m_coltype[i])); index+= m_table->s->field[i]->save_field_metadata(&m_field_metadata[index]); + DBUG_EXECUTE_IF("inject_invalid_blob_size", + { + if (m_coltype[i] == MYSQL_TYPE_BLOB) + m_field_metadata[index-1] = 5; + }); } DBUG_RETURN(index); } @@ -5941,7 +5946,9 @@ Table_map_log_event::Table_map_log_event(THD *thd, TABLE *tbl, ulong tid, m_field_metadata(0), m_field_metadata_size(0), m_null_bits(0), - m_meta_memory(NULL) + m_meta_memory(NULL), + m_optional_metadata_len(0), + m_optional_metadata(NULL) { uchar cbuf[MAX_INT_WIDTH]; uchar *cbuf_end; @@ -5975,6 +5982,7 @@ Table_map_log_event::Table_map_log_event(THD *thd, TABLE *tbl, ulong tid, m_coltype= reinterpret_cast<uchar*>(m_memory); for (unsigned int i= 0 ; i < m_table->s->fields ; ++i) m_coltype[i]= m_table->field[i]->binlog_type(); + DBUG_EXECUTE_IF("inject_invalid_column_type", m_coltype[1]= 230;); } /* @@ -6013,6 +6021,9 @@ Table_map_log_event::Table_map_log_event(THD *thd, TABLE *tbl, ulong tid, if (m_table->field[i]->maybe_null()) m_null_bits[(i / 8)]+= 1 << (i % 8); + init_metadata_fields(); + m_data_size+= m_metadata_buf.length(); + DBUG_VOID_RETURN; } @@ -6307,9 +6318,414 @@ bool Table_map_log_event::write_data_body() write_data(m_coltype, m_colcnt) || write_data(mbuf, (size_t) (mbuf_end - mbuf)) || write_data(m_field_metadata, m_field_metadata_size), - write_data(m_null_bits, (m_colcnt + 7) / 8); + write_data(m_null_bits, (m_colcnt + 7) / 8) || + write_data((const uchar*) m_metadata_buf.ptr(), + m_metadata_buf.length()); } +/** + stores an integer into packed format. + + @param[out] str_buf a buffer where the packed integer will be stored. + @param[in] length the integer will be packed. + */ +static inline +void store_compressed_length(String &str_buf, ulonglong length) +{ + // Store Type and packed length + uchar buf[4]; + uchar *buf_ptr = net_store_length(buf, length); + + str_buf.append(reinterpret_cast<char *>(buf), buf_ptr-buf); +} + +/** + Write data into str_buf with Type|Length|Value(TLV) format. + + @param[out] str_buf a buffer where the field is stored. + @param[in] type type of the field + @param[in] length length of the field value + @param[in] value value of the field +*/ +static inline +bool write_tlv_field(String &str_buf, + enum Table_map_log_event::Optional_metadata_field_type + type, uint length, const uchar *value) +{ + /* type is stored in one byte, so it should never bigger than 255. */ + DBUG_ASSERT(static_cast<int>(type) <= 255); + str_buf.append((char) type); + store_compressed_length(str_buf, length); + return str_buf.append(reinterpret_cast<const char *>(value), length); +} + +/** + Write data into str_buf with Type|Length|Value(TLV) format. + + @param[out] str_buf a buffer where the field is stored. + @param[in] type type of the field + @param[in] value value of the field +*/ +static inline +bool write_tlv_field(String &str_buf, + enum Table_map_log_event::Optional_metadata_field_type + type, const String &value) +{ + return write_tlv_field(str_buf, type, value.length(), + reinterpret_cast<const uchar *>(value.ptr())); +} + + +#ifdef MYSQL_SERVER +static inline bool is_numeric_field(const Field *field) +{ + return is_numeric_type(field->binlog_type()); +} + +static inline bool is_character_field(const Field *field) +{ + return is_character_type(field->real_type()); +} + +static inline bool is_enum_field(const Field *field) +{ + return field->real_type() == MYSQL_TYPE_ENUM; +} + +static inline bool is_set_field(const Field *field) +{ + return field->real_type() == MYSQL_TYPE_SET; +} + +static inline bool is_geometry_field(const Field *field) +{ + return field->real_type() == MYSQL_TYPE_GEOMETRY; +} + +static inline bool is_enum_or_set_field(const Field *field) { + return is_enum_or_set_type(field->real_type()); +} + + +void Table_map_log_event::init_metadata_fields() +{ + DBUG_ENTER("init_metadata_fields"); + DBUG_EXECUTE_IF("simulate_no_optional_metadata", DBUG_VOID_RETURN;); + + if (init_signedness_field() || + init_charset_field(&is_character_field, DEFAULT_CHARSET, + COLUMN_CHARSET) || + init_geometry_type_field()) + { + m_metadata_buf.length(0); + DBUG_VOID_RETURN; + } + + if (binlog_row_metadata == BINLOG_ROW_METADATA_FULL) + { + if (DBUG_EVALUATE_IF("dont_log_column_name", 0, init_column_name_field()) || + init_charset_field(&is_enum_or_set_field, ENUM_AND_SET_DEFAULT_CHARSET, + ENUM_AND_SET_COLUMN_CHARSET) || + init_set_str_value_field() || + init_enum_str_value_field() || + init_primary_key_field()) + m_metadata_buf.length(0); + } + DBUG_VOID_RETURN; +} + +bool Table_map_log_event::init_signedness_field() +{ + /* use it to store signed flags, each numeric column take a bit. */ + StringBuffer<128> buf; + unsigned char flag= 0; + unsigned char mask= 0x80; + + for (unsigned int i= 0 ; i < m_table->s->fields ; ++i) + { + if (is_numeric_field(m_table->field[i])) + { + Field_num *field= dynamic_cast<Field_num *>(m_table->field[i]); + + if (field->unsigned_flag) + flag|= mask; + + mask >>= 1; + + // 8 fields are tested, store the result and clear the flag. + if (mask == 0) + { + buf.append(flag); + flag= 0; + mask= 0x80; + } + } + } + + // Stores the signedness flags of last few columns + if (mask != 0x80) + buf.append(flag); + + // The table has no numeric column, so don't log SIGNEDNESS field + if (buf.is_empty()) + return false; + + return write_tlv_field(m_metadata_buf, SIGNEDNESS, buf); +} + +bool Table_map_log_event::init_charset_field( + std::function<bool(const Field *)> include_type, + Optional_metadata_field_type default_charset_type, + Optional_metadata_field_type column_charset_type) +{ + DBUG_EXECUTE_IF("simulate_init_charset_field_error", return true;); + + std::map<uint, uint> collation_map; + // For counting characters columns + uint char_col_cnt= 0; + + /* Find the collation number used by most fields */ + for (unsigned int i= 0 ; i < m_table->s->fields ; ++i) + { + if (include_type(m_table->field[i])) + { + Field_str *field= dynamic_cast<Field_str *>(m_table->field[i]); + + collation_map[field->charset()->number]++; + char_col_cnt++; + } + } + + if (char_col_cnt == 0) + return false; + + /* Find the most used collation */ + uint most_used_collation= 0; + uint most_used_count= 0; + for (std::map<uint, uint>::iterator it= collation_map.begin(); + it != collation_map.end(); it++) + { + if (it->second > most_used_count) + { + most_used_count= it->second; + most_used_collation= it->first; + } + } + + /* + Comparing length of COLUMN_CHARSET field and COLUMN_CHARSET_WITH_DEFAULT + field to decide which field should be logged. + + Length of COLUMN_CHARSET = character column count * collation id size. + Length of COLUMN_CHARSET_WITH_DEFAULT = + default collation_id size + count of columns not use default charset * + (column index size + collation id size) + + Assume column index just uses 1 byte and collation number also uses 1 byte. + */ + if (char_col_cnt * 1 < (1 + (char_col_cnt - most_used_count) * 2)) + { + StringBuffer<512> buf; + + /* + Stores character set information into COLUMN_CHARSET format, + character sets of all columns are stored one by one. + ----------------------------------------- + | Charset number | .... |Charset number | + ----------------------------------------- + */ + for (unsigned int i= 0 ; i < m_table->s->fields ; ++i) + { + if (include_type(m_table->field[i])) + { + Field_str *field= dynamic_cast<Field_str *>(m_table->field[i]); + + store_compressed_length(buf, field->charset()->number); + } + } + return write_tlv_field(m_metadata_buf, column_charset_type, buf); + } + else + { + StringBuffer<512> buf; + uint char_column_index= 0; + uint default_collation= most_used_collation; + + /* + Stores character set information into DEFAULT_CHARSET format, + First stores the default character set, and then stores the character + sets different to default character with their column index one by one. + -------------------------------------------------------- + | Default Charset | Col Index | Charset number | ... | + -------------------------------------------------------- + */ + + // Store the default collation number + store_compressed_length(buf, default_collation); + + for (unsigned int i= 0 ; i < m_table->s->fields ; ++i) + { + if (include_type(m_table->field[i])) + { + Field_str *field= dynamic_cast<Field_str *>(m_table->field[i]); + + if (field->charset()->number != default_collation) + { + store_compressed_length(buf, char_column_index); + store_compressed_length(buf, field->charset()->number); + } + char_column_index++; + } + } + return write_tlv_field(m_metadata_buf, default_charset_type, buf); + } +} + +bool Table_map_log_event::init_column_name_field() +{ + StringBuffer<2048> buf; + + for (unsigned int i= 0 ; i < m_table->s->fields ; ++i) + { + size_t len= strlen(m_table->field[i]->field_name.str); + + store_compressed_length(buf, len); + buf.append(m_table->field[i]->field_name.str, len); + } + return write_tlv_field(m_metadata_buf, COLUMN_NAME, buf); +} + +bool Table_map_log_event::init_set_str_value_field() +{ + StringBuffer<1024> buf; + + /* + SET string values are stored in the same format: + ---------------------------------------------- + | Value number | value1 len | value 1| .... | // first SET column + ---------------------------------------------- + | Value number | value1 len | value 1| .... | // second SET column + ---------------------------------------------- + */ + for (unsigned int i= 0 ; i < m_table->s->fields ; ++i) + { + if (is_set_field(m_table->field[i])) + { + TYPELIB *typelib= dynamic_cast<Field_set *>(m_table->field[i])->typelib; + + store_compressed_length(buf, typelib->count); + for (unsigned int i= 0; i < typelib->count; i++) + { + store_compressed_length(buf, typelib->type_lengths[i]); + buf.append(typelib->type_names[i], typelib->type_lengths[i]); + } + } + } + if (buf.length() > 0) + return write_tlv_field(m_metadata_buf, SET_STR_VALUE, buf); + return false; +} + +bool Table_map_log_event::init_enum_str_value_field() +{ + StringBuffer<1024> buf; + + /* ENUM is same to SET columns, see comment in init_set_str_value_field */ + for (unsigned int i= 0 ; i < m_table->s->fields ; ++i) + { + if (is_enum_field(m_table->field[i])) + { + TYPELIB *typelib= dynamic_cast<Field_enum *>(m_table->field[i])->typelib; + + store_compressed_length(buf, typelib->count); + for (unsigned int i= 0; i < typelib->count; i++) + { + store_compressed_length(buf, typelib->type_lengths[i]); + buf.append(typelib->type_names[i], typelib->type_lengths[i]); + } + } + } + + if (buf.length() > 0) + return write_tlv_field(m_metadata_buf, ENUM_STR_VALUE, buf); + return false; +} + +bool Table_map_log_event::init_geometry_type_field() +{ + StringBuffer<256> buf; + + /* Geometry type of geometry columns is stored one by one as packed length */ + for (unsigned int i= 0 ; i < m_table->s->fields ; ++i) + { + if (is_geometry_field(m_table->field[i])) + { + int type= dynamic_cast<Field_geom *>(m_table->field[i])->geom_type; + DBUG_EXECUTE_IF("inject_invalid_geometry_type", type= 100;); + store_compressed_length(buf, type); + } + } + + if (buf.length() > 0) + return write_tlv_field(m_metadata_buf, GEOMETRY_TYPE, buf); + return false; +} + +bool Table_map_log_event::init_primary_key_field() +{ + DBUG_EXECUTE_IF("simulate_init_primary_key_field_error", return true;); + + if (unlikely(m_table->s->primary_key == MAX_KEY)) + return false; + + // If any key column uses prefix like KEY(c1(10)) */ + bool has_prefix= false; + KEY *pk= m_table->key_info + m_table->s->primary_key; + + DBUG_ASSERT(pk->user_defined_key_parts > 0); + + /* Check if any key column uses prefix */ + for (uint i= 0; i < pk->user_defined_key_parts; i++) + { + KEY_PART_INFO *key_part= pk->key_part+i; + if (key_part->length != m_table->field[key_part->fieldnr-1]->key_length()) + { + has_prefix= true; + break; + } + } + + StringBuffer<128> buf; + + if (!has_prefix) + { + /* Index of PK columns are stored one by one. */ + for (uint i= 0; i < pk->user_defined_key_parts; i++) + { + KEY_PART_INFO *key_part= pk->key_part+i; + store_compressed_length(buf, key_part->fieldnr-1); + } + return write_tlv_field(m_metadata_buf, SIMPLE_PRIMARY_KEY, buf); + } + else + { + /* Index of PK columns are stored with a prefix length one by one. */ + for (uint i= 0; i < pk->user_defined_key_parts; i++) + { + KEY_PART_INFO *key_part= pk->key_part+i; + size_t prefix= 0; + + store_compressed_length(buf, key_part->fieldnr-1); + + // Store character length but not octet length + if (key_part->length != m_table->field[key_part->fieldnr-1]->key_length()) + prefix= key_part->length / key_part->field->charset()->mbmaxlen; + store_compressed_length(buf, prefix); + } + return write_tlv_field(m_metadata_buf, PRIMARY_KEY_WITH_PREFIX, buf); + } +} +#endif #if defined(HAVE_REPLICATION) /* diff --git a/sql/mysqld.cc b/sql/mysqld.cc index f3d36bfc15c..41c8ff38a1c 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -450,6 +450,7 @@ my_bool opt_noacl; my_bool sp_automatic_privileges= 1; ulong opt_binlog_rows_event_max_size; +ulong binlog_row_metadata; my_bool opt_master_verify_checksum= 0; my_bool opt_slave_sql_verify_checksum= 1; const char *binlog_format_names[]= {"MIXED", "STATEMENT", "ROW", NullS}; diff --git a/sql/mysqld.h b/sql/mysqld.h index 505741b5458..693aee5e743 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -244,6 +244,7 @@ extern ulonglong max_binlog_cache_size, max_binlog_stmt_cache_size; extern ulong max_binlog_size; extern ulong slave_max_allowed_packet; extern ulong opt_binlog_rows_event_max_size; +extern ulong binlog_row_metadata; extern ulong thread_cache_size; extern ulong stored_program_cache_size; extern ulong opt_slave_parallel_threads; diff --git a/sql/sql_class.h b/sql/sql_class.h index 63f964e96ce..b254e206ca0 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -137,6 +137,11 @@ enum enum_binlog_row_image { /** All columns in both before and after image. */ BINLOG_ROW_IMAGE_FULL= 2 }; +// Values for binlog_row_metadata sysvar +enum enum_binlog_row_metadata { + BINLOG_ROW_METADATA_MINIMAL= 0, + BINLOG_ROW_METADATA_FULL= 1 +}; /* Bits for different SQL modes modes (including ANSI mode) */ diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 7bc294fcc99..d7adce57c76 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -6167,6 +6167,18 @@ static Sys_var_enum Sys_binlog_row_image( SESSION_VAR(binlog_row_image), CMD_LINE(REQUIRED_ARG), binlog_row_image_names, DEFAULT(BINLOG_ROW_IMAGE_FULL)); +static const char *binlog_row_metadata_names[]= {"MINIMAL", "FULL", NullS}; +static Sys_var_enum Sys_binlog_row_metadata( + "binlog_row_metadata", + "Controls whether metadata is logged using FULL or MINIMAL format. " + "FULL causes all metadata to be logged; MINIMAL means that only " + "metadata actually required by slave is logged. Default: MINIMAL.", + GLOBAL_VAR(binlog_row_metadata), CMD_LINE(REQUIRED_ARG), + binlog_row_metadata_names, DEFAULT(BINLOG_ROW_METADATA_MINIMAL), + NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(NULL), + ON_UPDATE(NULL)); + + static bool check_pseudo_slave_mode(sys_var *self, THD *thd, set_var *var) { longlong previous_val= thd->variables.pseudo_slave_mode;