Hi, Rucha! Thanks for combining all commits in one and a cleanup. See comments below. On Mar 18, Rucha Deodhar wrote:

revision-id: 0fc221a012d (mariadb-10.5.2-402-g0fc221a012d) parent(s): a1542f8a573 author: Rucha Deodhar <rucha.deodhar@mariadb.com> committer: Rucha Deodhar <rucha.deodhar@mariadb.com> timestamp: 2021-03-16 23:31:12 +0530 message:

MDEV-8334: Rename utf8 to utf8mb3

This patch changes the main name of 3 byte character set from utf8 to utf8mb3. New old_mode UTF8_IS_UTF8MB3 is added and set TRUE by default, so that utf8 would mean utf8mb3. If not set, utf8 would mean utf8mb4.

diff --git a/client/mysqlcheck.c b/client/mysqlcheck.c index fb3103a318d..a8990d8cb6b 100644 --- a/client/mysqlcheck.c +++ b/client/mysqlcheck.c @@ -437,7 +437,7 @@ static int get_options(int *argc, char ***argv) if (!default_charset) { if (opt_fix_db_names || opt_fix_table_names) - default_charset= (char*) "utf8"; + default_charset= (char*) "utf8mb3";

why not to keep it utf8?

else default_charset= (char*) MYSQL_AUTODETECT_CHARSET_NAME; } diff --git a/client/mysqldump.c b/client/mysqldump.c index 7c363973da2..900456b31b2 100644 --- a/client/mysqldump.c +++ b/client/mysqldump.c @@ -3235,7 +3235,7 @@ static uint get_table_structure(const char *table, const char *db, char *table_t { fprintf(sql_file, "/*!40101 SET @saved_cs_client = @@character_set_client */;\n" - "/*!40101 SET character_set_client = utf8 */;\n" + "/*!40101 SET character_set_client = utf8mb3 */;\n"

why not to keep it utf8?

"%s%s;\n" "/*!40101 SET character_set_client = @saved_cs_client */;\n", is_log_table ? "CREATE TABLE IF NOT EXISTS " : "", diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index 3083326a7e0..c62252257b9 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -117,7 +117,7 @@ xb_mysql_connect() mysql_options(connection, MYSQL_PLUGIN_DIR, xb_plugin_dir); } mysql_options(connection, MYSQL_OPT_PROTOCOL, &opt_protocol); - mysql_options(connection,MYSQL_SET_CHARSET_NAME, "utf8"); + mysql_options(connection,MYSQL_SET_CHARSET_NAME, "utf8mb3");

why not to keep it utf8?

msg("Connecting to MySQL server host: %s, user: %s, password: %s, " "port: %s, socket: %s", opt_host ? opt_host : "localhost", @@ -1506,7 +1506,7 @@ write_xtrabackup_info(MYSQL *connection, const char * filename, bool history, "incremental ENUM('Y', 'N') DEFAULT NULL," "format ENUM('file', 'tar', 'xbstream') DEFAULT NULL," "compressed ENUM('Y', 'N') DEFAULT NULL" - ") CHARACTER SET utf8 ENGINE=innodb", false); + ") CHARACTER SET utf8mb3 ENGINE=innodb", false);

why not to keep it utf8?

#define ESCAPE_BOOL(expr) ((expr)?"'Y'":"'N'") diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 62fdace654d..0f500c35ff6 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -1716,7 +1716,7 @@ static int create_bootstrap_file() if(!f) return -1;

- fputs("SET NAMES UTF8;\n",f); + fputs("SET NAMES UTF8MB3;\n",f);

why not to keep it utf8?

enumerate_ibd_files(append_export_table); for (std::set<std::string>::iterator it = tables_for_export.begin(); it != tables_for_export.end(); it++) diff --git a/mysql-test/main/create-uca.test b/mysql-test/main/create-uca.test index 0acb51f7286..f73f6114962 100644 --- a/mysql-test/main/create-uca.test +++ b/mysql-test/main/create-uca.test @@ -1,5 +1,5 @@ # Prerequisites -let collation=utf8_unicode_ci; +let collation=utf8mb3_unicode_ci;

that's fine, as we generally don't want tests to depend on the old_mode.

--source include/have_collation.inc

# Initial cleanup diff --git a/mysql-test/main/ctype_ldml.result b/mysql-test/main/ctype_ldml.result index 22b7a316111..7c284520733 100644 --- a/mysql-test/main/ctype_ldml.result +++ b/mysql-test/main/ctype_ldml.result @@ -8,7 +8,6 @@ Variable_name Value character_sets_dir MYSQL_TEST_DIR/std_data/ldml/ show collation like 'utf8_phone_ci'; Collation Charset Id Default Compiled Sortlen -utf8_phone_ci utf8 352 8

I suppose the test should show there is a _phone_ci collation. As you've renamed it, you need to adjust the test to do show collation like 'utf8mb3_phone_ci';

CREATE TABLE t1 ( name VARCHAR(64), phone VARCHAR(64) CHARACTER SET utf8 COLLATE utf8_phone_ci @@ -37,7 +36,6 @@ Bar +7-912-800-80-01 DROP TABLE t1; show collation like 'utf8_test_ci'; Collation Charset Id Default Compiled Sortlen -utf8_test_ci utf8 353 8

same

create table t1 (c1 char(1) character set utf8 collate utf8_test_ci); insert into t1 values ('a'); select * from t1 where c1='b'; @@ -526,7 +524,6 @@ DROP TABLE t1; SET NAMES utf8 COLLATE utf8_phone_ci; SHOW COLLATION LIKE 'utf8_phone_ci'; Collation Charset Id Default Compiled Sortlen -utf8_phone_ci utf8 352 8

and here

SET NAMES utf8; SELECT hex(weight_string(_utf8mb4'a' collate utf8mb4_test_400_ci)); hex(weight_string(_utf8mb4'a' collate utf8mb4_test_400_ci)) diff --git a/mysql-test/main/show_check.result b/mysql-test/main/show_check.result index d031c792922..1c85ef596c9 100644 --- a/mysql-test/main/show_check.result +++ b/mysql-test/main/show_check.result @@ -874,12 +874,11 @@ set names utf8; ---------------------------------------------------------------- SHOW CHARACTER SET LIKE 'utf8'; Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr -def information_schema CHARACTER_SETS CHARACTER_SETS CHARACTER_SET_NAME Charset 253 96 4 N 1 0 33 -def information_schema CHARACTER_SETS CHARACTER_SETS DESCRIPTION Description 253 180 13 N 1 0 33 -def information_schema CHARACTER_SETS CHARACTER_SETS DEFAULT_COLLATE_NAME Default collation 253 96 15 N 1 0 33 -def information_schema CHARACTER_SETS CHARACTER_SETS MAXLEN Maxlen 8 3 1 N 32769 0 63 +def information_schema CHARACTER_SETS CHARACTER_SETS CHARACTER_SET_NAME Charset 253 96 0 N 1 0 33 +def information_schema CHARACTER_SETS CHARACTER_SETS DESCRIPTION Description 253 180 0 N 1 0 33 +def information_schema CHARACTER_SETS CHARACTER_SETS DEFAULT_COLLATE_NAME Default collation 253 96 0 N 1 0 33 +def information_schema CHARACTER_SETS CHARACTER_SETS MAXLEN Maxlen 8 3 0 N 32769 0 63 Charset Description Default collation Maxlen -utf8 UTF-8 Unicode utf8_general_ci 3

again, I suspect this test should now do `SHOW CHARACTER SET LIKE 'utf8mb3';

---------------------------------------------------------------- SHOW COLLATION LIKE 'latin1_bin'; Catalog Database Table Table_alias Column Column_alias Type Length Max length Is_null Flags Decimals Charsetnr diff --git a/mysql-test/suite/funcs_1/r/charset_collation.result b/mysql-test/suite/funcs_1/r/charset_collation.result index 31bd30c5acf..6b52e80d6ba 100644 --- a/mysql-test/suite/funcs_1/r/charset_collation.result +++ b/mysql-test/suite/funcs_1/r/charset_collation.result @@ -9,7 +9,6 @@ ORDER BY character_set_name; CHARACTER_SET_NAME DEFAULT_COLLATE_NAME DESCRIPTION MAXLEN binary binary Binary pseudo charset 1 latin1 latin1_swedish_ci cp1252 West European 1 -utf8 utf8_general_ci UTF-8 Unicode 3

and here too, changing the test is in order

SELECT * FROM information_schema.collations diff --git a/mysql-test/suite/funcs_1/r/is_column_privileges.result b/mysql-test/suite/funcs_1/r/is_column_privileges.result index b6be9118048..46b2d515041 100644 --- a/mysql-test/suite/funcs_1/r/is_column_privileges.result +++ b/mysql-test/suite/funcs_1/r/is_column_privileges.result @@ -45,7 +45,7 @@ COLUMN_PRIVILEGES CREATE TEMPORARY TABLE `COLUMN_PRIVILEGES` ( `COLUMN_NAME` varchar(64) NOT NULL DEFAULT '', `PRIVILEGE_TYPE` varchar(64) NOT NULL DEFAULT '', `IS_GRANTABLE` varchar(3) NOT NULL DEFAULT '' -) ENGINE=MEMORY DEFAULT CHARSET=utf8 +) ENGINE=MEMORY DEFAULT CHARSET=utf8mb3

Just a thought. Did you also fix all --embedded, all --ps, and all --big --big tests to pass?

SHOW COLUMNS FROM information_schema.COLUMN_PRIVILEGES; Field Type Null Key Default Extra GRANTEE varchar(190) NO diff --git a/mysql-test/suite/innodb/r/innodb_ctype_ldml.result b/mysql-test/suite/innodb/r/innodb_ctype_ldml.result index 502f57156c3..8a96c023134 100644 --- a/mysql-test/suite/innodb/r/innodb_ctype_ldml.result +++ b/mysql-test/suite/innodb/r/innodb_ctype_ldml.result @@ -8,7 +8,6 @@ Variable_name Value character_sets_dir MYSQL_TEST_DIR/std_data/ldml/ show collation like 'utf8_phone_ci'; Collation Charset Id Default Compiled Sortlen -utf8_phone_ci utf8 352 8

deja vu. I rememeber I've commented on this very thing already :) please, update this test too.

CREATE TABLE t1 ( name VARCHAR(64), phone VARCHAR(64) CHARACTER SET utf8 COLLATE utf8_phone_ci Binary files a/mysql-test/suite/sys_vars/r/character_set_results_basic.result and b/mysql-test/suite/sys_vars/r/character_set_results_basic.result differ diff --git a/mysql-test/suite/sys_vars/r/old_mode_basic.result b/mysql-test/suite/sys_vars/r/old_mode_basic.result index 39c8e554be2..a6b95f1c60c 100644 --- a/mysql-test/suite/sys_vars/r/old_mode_basic.result +++ b/mysql-test/suite/sys_vars/r/old_mode_basic.result @@ -167,8 +167,100 @@ NO_PROGRESS_INFO SET @@global.old_mode = @global_start_value; SELECT @@global.old_mode; @@global.old_mode - +UTF8_IS_UTF8MB3 SET @@session.old_mode = @session_start_value; SELECT @@session.old_mode; @@session.old_mode - +UTF8_IS_UTF8MB3 +# +# Beginning of 10.6 test +# +# MDEV-8334: Rename utf8 to utf8mb3 +#

Ah! Great.

+# Save and display old values +SET @save_old_mode = @@OLD_MODE; +SET @save_character_set_server = @@character_set_server; +SET @save_character_set_client = @@character_set_client; +SET @save_character_set_results = @@character_set_results; +SET @save_character_set_connection = @@character_set_connection; +SET @save_character_set_filesystem = @@character_set_filesystem; +SET @save_character_set_database = @@character_set_database; +SET @save_collation_connection = @@collation_connection; +SET @save_collation_server = @@collation_server; +SET @save_collation_database = @@collation_database; +SELECT @@OLD_MODE; +@@OLD_MODE +UTF8_IS_UTF8MB3 +SELECT @@character_set_server,@@character_set_client,@@character_set_results, +@@character_set_connection, @@character_set_filesystem, @@character_set_database, +@@collation_connection, @@collation_server, @@collation_database; +@@character_set_server @@character_set_client @@character_set_results @@character_set_connection @@character_set_filesystem @@character_set_database @@collation_connection @@collation_server @@collation_database +latin1 latin1 latin1 latin1 binary latin1 latin1_swedish_ci latin1_swedish_ci latin1_swedish_ci +# +# UTF8MB3 alias for UTF8 +# +SET @@character_set_server = utf8; +SET @@character_set_client = utf8; +SET @@character_set_results = utf8; +SET @@character_set_connection = utf8; +SET @@character_set_filesystem = utf8; +SET @@character_set_database = utf8; +SET @@collation_connection = utf8_general_ci; +SET @@collation_server = utf8_unicode_ci; +SET @@collation_database = utf8_bin; +SELECT @@character_set_server, @@character_set_client, @@character_set_results, +@@character_set_connection, @@character_set_filesystem, @@character_set_database, +@@collation_connection, @@collation_server, @@collation_database; +@@character_set_server @@character_set_client @@character_set_results @@character_set_connection @@character_set_filesystem @@character_set_database @@collation_connection @@collation_server @@collation_database +utf8mb3 utf8mb3 utf8mb3 utf8mb3 utf8mb3 utf8mb3 utf8mb3_general_ci utf8mb3_unicode_ci utf8mb3_bin +CREATE DATABASE db1 CHARACTER SET = 'utf8' COLLATE = 'utf8_general_ci'; +ALTER DATABASE db1 CHARACTER SET = 'utf8' COLLATE = 'utf8_unicode_ci'; +CREATE TABLE tb1 (id1 INT) CHARACTER SET 'utf8' COLLATE 'utf8_bin'; +SHOW CREATE TABLE tb1; +Table Create Table +tb1 CREATE TABLE `tb1` ( + `id1` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_bin +DROP TABLE tb1; +DROP DATABASE db1; +# +# UTF8MB4 is alias for UTF8 +# +SET @@OLD_MODE=0; +SET @@character_set_server = utf8; +SET @@character_set_client = utf8; +SET @@character_set_results = utf8; +SET @@character_set_connection = utf8; +SET @@character_set_filesystem = utf8; +SET @@character_set_database = utf8; +SET @@collation_connection = utf8_general_ci; +SET @@collation_server = utf8_unicode_ci; +SET @@collation_database = utf8_bin; +SELECT @@character_set_server, @@character_set_client, @@character_set_results, +@@character_set_connection, @@character_set_filesystem, @@character_set_database, +@@collation_connection, @@collation_server, @@collation_database; +@@character_set_server @@character_set_client @@character_set_results @@character_set_connection @@character_set_filesystem @@character_set_database @@collation_connection @@collation_server @@collation_database +utf8mb4 utf8mb4 utf8mb4 utf8mb4 utf8mb4 utf8mb4 utf8mb4_general_ci utf8mb4_unicode_ci utf8mb4_bin +CREATE DATABASE db1 CHARACTER SET = 'utf8' COLLATE = 'utf8_general_ci'; +ALTER DATABASE db1 CHARACTER SET = 'utf8' COLLATE = 'utf8_unicode_ci'; +CREATE TABLE tb1 (id1 INT) CHARACTER SET 'utf8' COLLATE 'utf8_bin'; +SHOW CREATE TABLE tb1; +Table Create Table +tb1 CREATE TABLE `tb1` ( + `id1` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin +DROP TABLE tb1; +DROP DATABASE db1; +SET @@OLD_MODE = @save_old_mode; +SET @@character_set_server = @save_character_set_server; +SET @@character_set_client = @save_character_set_client; +SET @@character_set_results = @save_character_set_results; +SET @@character_set_connection = @save_character_set_connection; +SET @@character_set_filesystem = @save_character_set_filesystem; +SET @@character_set_database = @save_character_set_database; +SET @@collation_connection = @save_collation_connection; +SET @@collation_server = @save_collation_server; +SET @@collation_database = @save_collation_database; +# +# End of 10.6 test +# diff --git a/mysys/charset.c b/mysys/charset.c index 32cfeb56e2d..dbb2749d217 100644 --- a/mysys/charset.c +++ b/mysys/charset.c @@ -763,22 +764,13 @@ get_charset_number_internal(const char *charset_name, uint cs_flags) }

-static const char* -get_charset_name_alias(const char *name) -{ - if (!my_strcasecmp(&my_charset_latin1, name, "utf8mb3")) - return "utf8"; - return NULL; -} - - uint get_charset_number(const char *charset_name, uint cs_flags) { uint id; my_pthread_once(&charsets_initialized, init_available_charsets); if ((id= get_charset_number_internal(charset_name, cs_flags))) return id; - if ((charset_name= get_charset_name_alias(charset_name))) + if ((charset_name= !my_strcasecmp(&my_charset_latin1, charset_name, "utf8") ? "utf8mb3" : NULL))

Huh? Why do you not check MY_UTF8_IS_UTF8MB3 here?

return get_charset_number_internal(charset_name, cs_flags); return 0; } @@ -820,7 +812,7 @@ static CHARSET_INFO *find_collation_data_inheritance_source(CHARSET_INFO *cs) char name[MY_CS_NAME_SIZE + 1]; memcpy(name, beg, end - beg); name[end - beg]= '\0'; - return inheritance_source_by_id(cs, get_collation_number(name)); + return inheritance_source_by_id(cs, get_collation_number(name,MYF(0)));

and not here?

} return NULL; } @@ -961,7 +953,28 @@ my_collation_get_by_name(MY_CHARSET_LOADER *loader, CHARSET_INFO *get_charset_by_name(const char *cs_name, myf flags) { MY_CHARSET_LOADER loader; + my_bool utf8_is_utf8mb3= flags & MY_UTF8_IS_UTF8MB3 ? 1 : 0; + char *copy_of_name= (char*)cs_name; + char start[6], result[64]; + char *temp_cs_name; + my_charset_loader_init_mysys(&loader); + + if (!strcasecmp("utf8",copy_of_name)) + cs_name = (const char*)(utf8_is_utf8mb3 ? "utf8mb3" : "utf8mb4"); + + strncpy(start, cs_name, 5); + temp_cs_name= (char *)(utf8_is_utf8mb3 ? "utf8mb3_":"utf8mb4_"); + + if (!strncasecmp("utf8_", start,5)) + { + copy_of_name+= 5; + result[63]='\0'; + strcpy(result, temp_cs_name); + strcat(result, copy_of_name); + result[strlen(copy_of_name)+strlen(temp_cs_name)]='\0'; + cs_name= (const char *) result; + }

And why do you do all that ^^^ ? Old code didn't try to change utf8mb3 to utf8 here, because, I suppose, my_collation_get_by_name() below did all that. Why did you add alias resolution where none was?

return my_collation_get_by_name(&loader, cs_name, flags); }

@@ -1005,12 +1018,16 @@ get_charset_by_csname(const char *cs_name, uint cs_flags, myf flags) { MY_CHARSET_LOADER loader; my_charset_loader_init_mysys(&loader); + + if (!strcasecmp("utf8",cs_name)) + cs_name= (const char*)(flags & MY_UTF8_IS_UTF8MB3 ? "utf8mb3" : "utf8mb4");

same here

+ return my_charset_get_by_name(&loader, cs_name, cs_flags, flags); }

/** - Resolve character set by the character set name (utf8, latin1, ...). + Resolve character set by the character set name (utf8mb3, latin1, ...).

The function tries to resolve character set by the specified name. If there is character set with the given name, it is assigned to the "cs" @@ -1453,8 +1472,8 @@ static const MY_CSET_OS_NAME charsets[] =

{"US-ASCII", "latin1", my_cs_approx},

- {"utf8", "utf8", my_cs_exact}, - {"utf-8", "utf8", my_cs_exact}, + {"utf8mb3", "utf8mb3", my_cs_exact},

eh, no. Try to understand what this array is for.

+ {"utf-8", "utf8mb3", my_cs_exact}, #endif {NULL, NULL, 0} }; diff --git a/plugin/handler_socket/client/hslongrun.cpp b/plugin/handler_socket/client/hslongrun.cpp index b7c02951340..7f88d48fff2 100644 --- a/plugin/handler_socket/client/hslongrun.cpp +++ b/plugin/handler_socket/client/hslongrun.cpp @@ -897,7 +897,7 @@ hs_longrun_init_table(const config& conf, int num_prepare, "v1 varchar(32) not null," "v2 varchar(32) not null," "v3 varchar(32) not null" - ") character set utf8 collate utf8_bin engine = innodb"); + ") character set utf8mb3 collate utf8_bin engine = innodb");

just keep it utf8

for (int i = 0; i < num_prepare; ++i) { const std::string i_str = to_stdstring(i); const std::string v1 = "pv1_" + i_str; diff --git a/plugin/handler_socket/client/hstest.pl b/plugin/handler_socket/client/hstest.pl index 1363e153c44..5924d8a0ce5 100755 --- a/plugin/handler_socket/client/hstest.pl +++ b/plugin/handler_socket/client/hstest.pl @@ -52,7 +52,7 @@ for my $action (@actions) { "k $keytype primary key" . ",v varchar(32) not null" . $moreflds . - ") character set utf8 collate utf8_bin " . + ") character set utf8mb3 collate utf8_bin " .

and here. forget about handlersocket

"engine = $engine"); } elsif ($action eq "insert") { print("INSERT $db.$table tablesize=$tablesize\n"); diff --git a/plugin/win_auth_client/common.cc b/plugin/win_auth_client/common.cc index 8b7319252ac..ddd34aec7da 100644 --- a/plugin/win_auth_client/common.cc +++ b/plugin/win_auth_client/common.cc @@ -384,7 +384,7 @@ char* wchar_to_utf8(const wchar_t *string, size_t *len) buf= (char*)malloc(buf_len + 1); if (!buf) { - DBUG_PRINT("error",("Out of memory when converting string '%S' to utf8", + DBUG_PRINT("error",("Out of memory when converting string '%S' to utf8mb3",

Nope, see what this function is doing.

string)); return NULL; } @@ -408,7 +408,7 @@ char* wchar_to_utf8(const wchar_t *string, size_t *len)

#ifndef DBUG_OFF Error_message_buf error_buf; - DBUG_PRINT("error", ("Could not convert string '%S' to utf8" + DBUG_PRINT("error", ("Could not convert string '%S' to utf8mb3"

same

", WideCharToMultiByte() failed with error %X (%s)", string, GetLastError(), get_last_error_message(error_buf))); @@ -451,7 +451,7 @@ wchar_t* utf8_to_wchar(const char *string, size_t *len)

if (!buf) { - DBUG_PRINT("error",("Out of memory when converting utf8 string '%s'" + DBUG_PRINT("error",("Out of memory when converting utf8mb3 string '%s'"

same

" to wide-char representation", string)); return NULL; } diff --git a/scripts/fill_help_tables.sql b/scripts/fill_help_tables.sql index d0efb750330..ad7c4fce9a4 100644 --- a/scripts/fill_help_tables.sql +++ b/scripts/fill_help_tables.sql @@ -22,7 +22,7 @@

don't change help tables, please. they'll be regenerated from the documentation

-- mysql -u root -p mysql < file_name

-set names 'utf8'; +set names 'utf8mb3';

set sql_log_bin = 0;

diff --git a/scripts/mysql_system_tables.sql b/scripts/mysql_system_tables.sql index e390f36a98b..7c8532577a1 100644 --- a/scripts/mysql_system_tables.sql +++ b/scripts/mysql_system_tables.sql @@ -209,7 +209,7 @@ SET @create_transaction_registry="CREATE TABLE IF NOT EXISTS transaction_registr UNIQUE KEY (commit_id), INDEX (begin_timestamp), INDEX (commit_timestamp, transaction_id) -) ENGINE=INNODB DEFAULT CHARSET=utf8 COLLATE=utf8_bin STATS_PERSISTENT=0"; +) ENGINE=INNODB DEFAULT CHARSET=utf8mb3 COLLATE=utf8_bin STATS_PERSISTENT=0";

here and in all other places in this file and other .sql files: you need to change the collation too, not just the charset.

SET @str=IF(@have_innodb <> 0, @create_innodb_table_stats, "SET @dummy = 0"); PREPARE stmt FROM @str; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 0bf21e02002..cc7568990b4 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -4039,7 +4039,10 @@ static int init_common_variables() *next_character_set_name++= '\0'; if (!(default_charset_info= get_charset_by_csname(default_character_set_name, - MY_CS_PRIMARY, MYF(MY_WME)))) + MY_CS_PRIMARY, + global_system_variables.old_behavior & + OLD_MODE_UTF8_IS_UTF8MB3 ? + MYF(MY_UTF8_IS_UTF8MB3 | MY_WME) : MYF(MY_WME))))

may be you'd better get this multi-line assignment out of if() ? e.g. myf utf8_flag= global_system_variables.old_behavior & OLD_MODE_UTF8_IS_UTF8MB3 : 0; default_charset_info= get_charset_by_csname(default_character_set_name, MY_CS_PRIMARY, MYF(utf8_flag | MY_WME)); if (!default_charset_info) ... or add a helper thd->utf8_alias() as I suggested below

{ if (next_character_set_name) { @@ -4056,7 +4059,10 @@ static int init_common_variables() if (default_collation_name) { CHARSET_INFO *default_collation; - default_collation= get_charset_by_name(default_collation_name, MYF(0)); + default_collation= get_charset_by_name(default_collation_name, + global_system_variables.old_behavior & + OLD_MODE_UTF8_IS_UTF8MB3 ? + MYF(MY_UTF8_IS_UTF8MB3) : MYF(0));

and here you'll be able to use utf8_flag without the conditional operator

if (!default_collation) { #ifdef WITH_PERFSCHEMA_STORAGE_ENGINE @@ -4097,7 +4103,10 @@ static int init_common_variables()

if (!(character_set_filesystem= get_charset_by_csname(character_set_filesystem_name, - MY_CS_PRIMARY, MYF(MY_WME)))) + MY_CS_PRIMARY, + global_system_variables.old_behavior & + OLD_MODE_UTF8_IS_UTF8MB3 ? + MYF(MY_UTF8_IS_UTF8MB3 | MY_WME) : MYF(MY_WME))))

and here

return 1; global_system_variables.character_set_filesystem= character_set_filesystem;

@@ -7415,7 +7424,9 @@ static void usage(void) DBUG_ENTER("usage"); if (!(default_charset_info= get_charset_by_csname(default_character_set_name, MY_CS_PRIMARY, - MYF(MY_WME)))) + global_system_variables.old_behavior & + OLD_MODE_UTF8_IS_UTF8MB3 ? + MYF(MY_UTF8_IS_UTF8MB3 | MY_WME) : MYF(MY_WME))))

split this one too?

exit(1); if (!default_collation_name) default_collation_name= (char*) default_charset_info->name; diff --git a/sql/sql_class.h b/sql/sql_class.h index 50b746fe514..15aa9ca8199 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -1007,6 +1008,36 @@ inline void update_global_memory_status(int64 size) my_atomic_add64_explicit(ptr, size, MY_MEMORY_ORDER_RELAXED); }

+inline const char* get_alias_collation_or_charset_name(const char* name, + bool utf8_is_utf8mb3) +{ + char *copy_of_name= (char*)name; + char start[6], result[64]; + char *temp_cs_name; + + if (!strchr(name,'_')) + { + if (!strcasecmp("utf8",name)) + name = utf8_is_utf8mb3 ? "utf8mb3" : "utf8mb4"; + return name; + } + else + { + strncpy(start, name, 5); + temp_cs_name= (char *)(utf8_is_utf8mb3 ? "utf8mb3_":"utf8mb4_"); + if (!strncasecmp("utf8_", start,5)) + { + copy_of_name+= 5; + result[63]='\0'; + strcpy(result, temp_cs_name); + strcat(result, copy_of_name); + result[strlen(copy_of_name)+strlen(temp_cs_name)]='\0'; + strcpy((char*)name,result); + } + } + return name; +}

Please, no. First, you cannot just copy `result` into `name`, because `result` is longer, you'll overwrite whatever was in memory after name's value. Second, you don't need to resolve aliases here, charset code already does it, don't duplicate that. Just pass MY_UTF8_IS_UTF8MB3 down to my_collation_get_by_name() below.

+ /** Get collation by name, send error to client on failure. @param name Collation name diff --git a/sql/sql_db.cc b/sql/sql_db.cc index 9bf16220535..f471d8edc66 100644 --- a/sql/sql_db.cc +++ b/sql/sql_db.cc @@ -583,9 +583,14 @@ bool load_db_opt(THD *thd, const char *path, Schema_specification_st *create) default-collation commands. */ if (!(create->default_table_charset= - get_charset_by_csname(pos+1, MY_CS_PRIMARY, MYF(0))) && + get_charset_by_csname(pos+1, MY_CS_PRIMARY, + thd->variables.old_behavior & + OLD_MODE_UTF8_IS_UTF8MB3 ? + MYF(MY_UTF8_IS_UTF8MB3) : MYF(0))) && !(create->default_table_charset= - get_charset_by_name(pos+1, MYF(0)))) + get_charset_by_name(pos+1, thd->variables.old_behavior & + OLD_MODE_UTF8_IS_UTF8MB3 ? + MYF(MY_UTF8_IS_UTF8MB3) : MYF(0))))

again, please move myf flags manipulations out of if()

{ sql_print_error("Error while loading database options: '%s':",path); sql_print_error(ER_THD(thd, ER_UNKNOWN_CHARACTER_SET),pos+1); @@ -595,7 +600,9 @@ bool load_db_opt(THD *thd, const char *path, Schema_specification_st *create) else if (!strncmp(buf,"default-collation", (pos-buf))) { if (!(create->default_table_charset= get_charset_by_name(pos+1, - MYF(0)))) + thd->variables.old_behavior & + OLD_MODE_UTF8_IS_UTF8MB3 ? + MYF(MY_UTF8_IS_UTF8MB3) : MYF(0))))

and here

{ sql_print_error("Error while loading database options: '%s':",path); sql_print_error(ER_THD(thd, ER_UNKNOWN_COLLATION),pos+1); diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 6871699dc5b..ad322eda097 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -2789,7 +2789,10 @@ int Lex_input_stream::scan_ident_middle(THD *thd, Lex_ident_cli_st *str, body_utf8_append(m_cpp_text_start, m_cpp_tok_start + length); ErrConvString csname(str->str + 1, str->length - 1, &my_charset_bin); CHARSET_INFO *cs= get_charset_by_csname(csname.ptr(), - MY_CS_PRIMARY, MYF(0)); + MY_CS_PRIMARY, + thd->variables.old_behavior & + OLD_MODE_UTF8_IS_UTF8MB3 ? + MYF(MY_UTF8_IS_UTF8MB3) : MYF(0));

may be it could be a helper in thd, like THD::utf8_alias() { return variables.old_behavior & OLD_MODE_UTF8_IS_UTF8MB3 ? MY_UTF8_IS_UTF8MB3 : 0; }

if (cs) { *introducer= cs; diff --git a/storage/connect/mysql-test/connect/my.cnf b/storage/connect/mysql-test/connect/my.cnf index 6310772d01f..83f0aa8ab30 100644 --- a/storage/connect/mysql-test/connect/my.cnf +++ b/storage/connect/mysql-test/connect/my.cnf @@ -14,4 +14,4 @@ MASTER_MYSOCK= @mysqld.1.socket SLAVE_MYPORT= @mysqld.2.port SLAVE_MYSOCK= @mysqld.2.socket

-PGCLIENTENCODING= UTF8 +PGCLIENTENCODING= UTF8MB3

eh... really? Are you sure you've tested it and it worked? by the name of it I suspect it's a postgresql client encoding.

diff --git a/storage/connect/mysql-test/connect/t/odbc_postgresql.sql b/storage/connect/mysql-test/connect/t/odbc_postgresql.sql index 1c302294393..3c78120a7a2 100644 --- a/storage/connect/mysql-test/connect/t/odbc_postgresql.sql +++ b/storage/connect/mysql-test/connect/t/odbc_postgresql.sql @@ -4,13 +4,13 @@ -- Run this script as a admin user: -- psql -U postgres < odbc_postgresql.sql

-SET NAMES 'UTF8'; +SET NAMES 'UTF8MB3';

same, postgresql

DROP DATABASE IF EXISTS mtr; DROP USER IF EXISTS mtr;

CREATE USER mtr WITH PASSWORD 'mtr'; -CREATE DATABASE mtr OWNER=mtr ENCODING='UTF8'; +CREATE DATABASE mtr OWNER=mtr ENCODING='UTF8MB3';

same

GRANT ALL ON DATABASE mtr TO mtr; \c mtr SET role mtr; diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index e5164fcc4fa..a26a05c2cf9 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -330,7 +330,7 @@ fts_word_t* fts_word_init( /*==========*/ fts_word_t* word, /*!< in: word to initialize */ - byte* utf8, /*!< in: UTF-8 string */ + byte* utf8mb3, /*!< in: UTF-8 string */

don't rename variables, please

ulint len) /*!< in: length of string in bytes */ { mem_heap_t* heap = mem_heap_create(sizeof(fts_node_t)); diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/collation_utf8_unicode_ci_french.result b/storage/mroonga/mysql-test/mroonga/storage/r/collation_utf8_unicode_ci_french.result index 3f24de87035..3659aa5aee8 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/collation_utf8_unicode_ci_french.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/collation_utf8_unicode_ci_french.result @@ -7,5 +7,4 @@ FULLTEXT INDEX (content) INSERT INTO diaries VALUES ("Je suis un garçon."); SELECT * FROM diaries WHERE MATCH (content) AGAINST ("garcon"); content -Je suis un garçon.

looks like a bug

DROP TABLE diaries; diff --git a/storage/mroonga/mysql-test/mroonga/storage/r/collation_utf8_unicode_ci_japanese.result b/storage/mroonga/mysql-test/mroonga/storage/r/collation_utf8_unicode_ci_japanese.result index 94ef2608b81..79dac1e63a7 100644 --- a/storage/mroonga/mysql-test/mroonga/storage/r/collation_utf8_unicode_ci_japanese.result +++ b/storage/mroonga/mysql-test/mroonga/storage/r/collation_utf8_unicode_ci_japanese.result @@ -7,5 +7,4 @@ FULLTEXT INDEX (content) INSERT INTO diaries VALUES ("ひらがなとカタカナを覚えました。"); SELECT * FROM diaries WHERE MATCH (content) AGAINST ("かたかな"); content -ひらがなとカタカナを覚えました。

that too

DROP TABLE diaries; diff --git a/storage/mroonga/vendor/groonga/CMakeLists.txt b/storage/mroonga/vendor/groonga/CMakeLists.txt index d271d4c4eb9..fc134b81cde 100644 --- a/storage/mroonga/vendor/groonga/CMakeLists.txt +++ b/storage/mroonga/vendor/groonga/CMakeLists.txt @@ -268,7 +268,7 @@ if(UNIX) ac_check_funcs(pthread_condattr_setpshared) endif()

-option(GRN_WITH_NFKC "use NFKC based UTF8 normalization." ON) +option(GRN_WITH_NFKC "use NFKC based UTF8MB3 normalization." ON)

not here, please

if(WIN32) ac_check_headers(winsock2.h) diff --git a/storage/mroonga/vendor/groonga/benchmark/bench-nfkc.c b/storage/mroonga/vendor/groonga/benchmark/bench-nfkc.c index ebae95b273b..3997b933e87 100644 --- a/storage/mroonga/vendor/groonga/benchmark/bench-nfkc.c +++ b/storage/mroonga/vendor/groonga/benchmark/bench-nfkc.c @@ -83,11 +83,11 @@ static void bench_char_type(gpointer user_data) { uint64_t code_point; - char utf8[7]; + char utf8mb3[7];

don't rename variables

for (code_point = 1; code_point < MAX_UNICODE; code_point++) { - ucs2utf8(code_point, (unsigned char *)utf8); - grn_nfkc50_char_type(utf8); + ucs2utf8(code_point, (unsigned char *)utf8mb3); + grn_nfkc50_char_type(utf8mb3); } }

diff --git a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/tool/dump_difference_utf8.rb b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/tool/dump_difference_utf8.rb index 4b6fde8c7b0..ce20a2c5b40 100644 --- a/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/tool/dump_difference_utf8.rb +++ b/storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql/tool/dump_difference_utf8.rb @@ -36,8 +36,8 @@ parser.sorted_pages.each do |page, characters| next if base == sort n_differences += 1 utf8s = [base, upper, lower, sort] - formatted_code_points = utf8s.collect do |utf8| - "%#07x" % Unicode.from_utf8(utf8) + formatted_code_points = utf8s.collect do |utf8mb3| + "%#07x" % Unicode.from_utf8(utf8mb3)

just revert all changes under storage/mroonga/vendor/*

end if sort.bytesize > base.bytesize n_expanded_sort_characters += 1 diff --git a/storage/sphinx/mysql-test/sphinx/my.cnf b/storage/sphinx/mysql-test/sphinx/my.cnf index f60380b7171..22cc06914f4 100644 --- a/storage/sphinx/mysql-test/sphinx/my.cnf +++ b/storage/sphinx/mysql-test/sphinx/my.cnf @@ -7,7 +7,7 @@ xmlpipe_command = cat @ENV.MTR_SUITE_DIR/testdata.xml [index test1] source = src1 docinfo = extern -charset_type = utf-8 +charset_type = utf-8mb3

revert

path = @ENV.MYSQLTEST_VARDIR/searchd/test1

[indexer] diff --git a/storage/spider/mysql-test/spider/bg/my.cnf b/storage/spider/mysql-test/spider/bg/my.cnf index 246099c623e..39f5bd01c67 100644 --- a/storage/spider/mysql-test/spider/bg/my.cnf +++ b/storage/spider/mysql-test/spider/bg/my.cnf @@ -75,15 +75,15 @@ MASTER_1_MYSOCK= @mysqld.1.1.socket MASTER_1_ENGINE_TYPE= Spider #MASTER_1_ENGINE_TYPE= MyISAM MASTER_1_ENGINE= ENGINE=Spider -MASTER_1_CHARSET= DEFAULT CHARSET=utf8 +MASTER_1_CHARSET= DEFAULT CHARSET=utf8mb3

Make sure to update both charset and collation here and in all other .cnf files in the spider suite

MASTER_1_ENGINE2= ENGINE=MyISAM -MASTER_1_CHARSET2= DEFAULT CHARSET=utf8 -MASTER_1_CHARSET3= DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci +MASTER_1_CHARSET2= DEFAULT CHARSET=utf8mb3 +MASTER_1_CHARSET3= DEFAULT CHARSET=utf8mb3 COLLATE=utf8_unicode_ci SLAVE1_1_MYPORT= @mysqld.4.1.port SLAVE1_1_MYSOCK= @mysqld.4.1.socket SLAVE1_1_ENGINE_TYPE= MyISAM SLAVE1_1_ENGINE= ENGINE=MyISAM -SLAVE1_1_CHARSET= DEFAULT CHARSET=utf8 +SLAVE1_1_CHARSET= DEFAULT CHARSET=utf8mb3 USE_CHILD_GROUP2= 1 OUTPUT_CHILD_GROUP2= 0 CHILD2_1_MYPORT= @mysqld.2.1.port diff --git a/storage/spider/spd_init_query.h b/storage/spider/spd_init_query.h index 19b04d50b82..f12cef377e2 100644 --- a/storage/spider/spd_init_query.h +++ b/storage/spider/spd_init_query.h @@ -559,7 +559,7 @@ static LEX_STRING spider_init_queries[] = { " table_name char(64) not null default ''," " primary key (table_id)," " unique uk1(db_name, table_name)" - " ) engine=Aria transactional=1 default charset=utf8 collate=utf8_bin;" + " ) engine=Aria transactional=1 default charset=utf8mb3 collate=utf8_bin;"

always change both charset and collation (everywhere in this file)

" create table if not exists mysql.spider_rewrite_table_tables(" " table_id bigint unsigned not null," " partition_id bigint unsigned not null auto_increment," diff --git a/tests/mysql_client_test.c b/tests/mysql_client_test.c index 0043786d477..5d9213591fb 100644 --- a/tests/mysql_client_test.c +++ b/tests/mysql_client_test.c @@ -19236,7 +19236,7 @@ static void test_bug12337762() rc= mysql_query(mysql, "create table charset_tab("\ "txt1 varchar(32) character set Latin1,"\ "txt2 varchar(32) character set Latin1 collate latin1_bin,"\ - "txt3 varchar(32) character set utf8 collate utf8_bin"\ + "txt3 varchar(32) character set utf8mb3 collate utf8_bin"\

both charset and collation

")");

DIE_UNLESS(rc == 0);

Regards, Sergei VP of MariaDB Server Engineering and security@mariadb.org