Re: [Maria-developers] 7b20964dd240: MDEV-8334: Rename utf8 to utf8mb3

Hi, Rucha! On Apr 15, Rucha Deodhar wrote:

revision-id: 7b20964dd240 parent(s): e9a2c9e author: Rucha Deodhar timestamp: 2021-03-26 00:55:56 +0530 message:

MDEV-8334: Rename utf8 to utf8mb3

This patch is made as a part of MDEV-8334 to fix failing test in unit and main test suite so that utf8mb3 characterset is recognized. Failing tests: main.mysql_client_test main.mysql_client_test_comp unit.conc_basic-t unit.conc_charset unit.conc_connection

diff --git a/libmariadb/ma_charset.c b/libmariadb/ma_charset.c index ee4b0f47..307cd522 100644 --- a/libmariadb/ma_charset.c +++ b/libmariadb/ma_charset.c @@ -67,6 +67,10 @@ #include #endif

+#define IS_UTF8(c)\ +(!strcasecmp((c), "utf8") || !strcasecmp((c), "utf8mb3") ||\ + !strcasecmp((c), "utf8mb4") || !strcasecmp((c), "utf-8")) + /* +----------------------------------------------------------------------+ | PHP Version 5 | @@ -1269,7 +1275,7 @@ struct st_madb_os_charset MADB_OS_CHARSET[]= {"57010", "ISCII Gujarati", NULL, NULL, MADB_CS_UNSUPPORTED}, {"57011", "ISCII Punjabi", NULL, NULL, MADB_CS_UNSUPPORTED}, {"65000", "utf-7 Unicode (UTF-7)", NULL, NULL, MADB_CS_UNSUPPORTED}, - {"65001", "utf-8 Unicode (UTF-8)", "utf8", NULL, MADB_CS_EXACT}, + {"65001", "utf-8 Unicode (UTF-8)", "utf8mb3", NULL, MADB_CS_EXACT},

No, keep this utf8, it's still a valid charset name, the server can figure it out what to map it to.

/* non Windows */ #else /* iconv encodings */ @@ -1337,8 +1343,8 @@ struct st_madb_os_charset MADB_OS_CHARSET[]= {"gb2312", "GB2312", "gb2312", "GB2312", MADB_CS_EXACT}, {"gbk", "GBK", "gbk", "GBK", MADB_CS_EXACT}, {"georgianps", "Georgian", "geostd8", "GEORGIAN-PS", MADB_CS_EXACT}, - {"utf8", "UTF8", "utf8", "UTF-8", MADB_CS_EXACT}, - {"utf-8", "UTF8", "utf8", "UTF-8", MADB_CS_EXACT}, + {"utf8mb3", "UTF8MB3", "utf8mb3", "UTF-8", MADB_CS_EXACT}, + {"utf-8", "UTF8MB3", "utf8mb3", "UTF-8", MADB_CS_EXACT},

same here

#endif {NULL, NULL, NULL, NULL, 0} }; @@ -1361,8 +1367,8 @@ const char *madb_get_os_character_set() return MADB_DEFAULT_CHARSET_NAME; while (MADB_OS_CHARSET[i].identifier) { - if (MADB_OS_CHARSET[i].supported > MADB_CS_UNSUPPORTED && - strcasecmp(MADB_OS_CHARSET[i].identifier, p) == 0) + if ((MADB_OS_CHARSET[i].supported > MADB_CS_UNSUPPORTED && + strcasecmp(MADB_OS_CHARSET[i].identifier, p) == 0) || IS_UTF8(p))

why?

return MADB_OS_CHARSET[i].charset; i++; } diff --git a/unittest/libmariadb/basic-t.c b/unittest/libmariadb/basic-t.c index c22e6c2b..e2943964 100644 --- a/unittest/libmariadb/basic-t.c +++ b/unittest/libmariadb/basic-t.c @@ -310,7 +310,8 @@ static int use_utf8(MYSQL *my)

while ((row= mysql_fetch_row(res)) != NULL) { - FAIL_IF(strcmp(row[0], "utf8"), "wrong character set"); + FAIL_IF(strcmp(row[0], get_utf8_name(mysql_get_server_version(my),"utf8")), + "wrong character set");

technically, C/C is a separate project, can run on any server with any config file. So it'd be safer to check that row[0] starts from utf8 and not assume that it depends on a server version in a specific way.

} FAIL_IF(mysql_errno(my), mysql_error(my)); mysql_free_result(res); diff --git a/unittest/libmariadb/charset.c b/unittest/libmariadb/charset.c index 898b6dad..ffa877bc 100644 --- a/unittest/libmariadb/charset.c +++ b/unittest/libmariadb/charset.c @@ -71,14 +71,20 @@ int bug_8378(MYSQL *mysql) { int test_client_character_set(MYSQL *mysql) { MY_CHARSET_INFO cs; + char collation_name[19]; char *csname= (char*) "utf8"; char *csdefault= (char*)mysql_character_set_name(mysql);

+ strcpy(collation_name,(const char*)get_utf8_name(mysql_get_server_version(mysql), + "utf8_general_ci")); +

This one is simpler. It only tests that mysql_set_character_set() works. Just don't use utf8, make it test on something else, e.g. on latin2.

FAIL_IF(mysql_set_character_set(mysql, csname), mysql_error(mysql));

mysql_get_character_set_info(mysql, &cs);

- FAIL_IF(strcmp(cs.csname, "utf8") || strcmp(cs.name, "utf8_general_ci"), "Character set != UTF8"); + FAIL_IF(strcmp(cs.csname, get_utf8_name(mysql_get_server_version(mysql),"utf8")) || + strcmp(cs.name, collation_name), + "Wrong UTF8 characterset"); FAIL_IF(mysql_set_character_set(mysql, csdefault), mysql_error(mysql));

return OK; @@ -537,6 +544,9 @@ static int test_bug30472(MYSQL *mysql)

SKIP_MAXSCALE;

+ strcpy(collation_name,(const char*)get_utf8_name(mysql_get_server_version(mysql), + "utf8_general_ci")); +

same here, the bug is https://bugs.mysql.com/bug.php?id=30472 "libmysql doesn't reset charset, insert_id after succ. mysql_change_user() call" so, does not need utf8 specifically. Change it to some easier to use charset.

if (mysql_get_server_version(mysql) < 50100 || !is_mariadb) { diag("Test requires MySQL Server version 5.1 or above"); diff --git a/unittest/libmariadb/connection.c b/unittest/libmariadb/connection.c index 70d347ce..eb9b39bb 100644 --- a/unittest/libmariadb/connection.c +++ b/unittest/libmariadb/connection.c @@ -644,9 +644,8 @@ int test_conc26(MYSQL *unused __attribute__((unused)))

FAIL_IF(my_test_connect(mysql, hostname, "notexistinguser", "password", schema, port, NULL, CLIENT_REMEMBER_OPTIONS), "Error expected"); - - FAIL_IF(!mysql->options.charset_name || strcmp(mysql->options.charset_name, "utf8") != 0, - "expected charsetname=utf8"); + FAIL_IF(!mysql->options.charset_name || strcmp(mysql->options.charset_name, "utf8") != 0, + "Wrong utf8 characterset for this version");

again, CONC-26 is "CLIENT_REMEMBER_OPTIONS flag missing" it doesn't apparently need utf8 specifically, so just use a different non-default charset there.

mysql_close(mysql);

mysql= mysql_init(NULL); @@ -981,7 +980,8 @@ static int test_sess_track_db(MYSQL *mysql) printf("# SESSION_TRACK_VARIABLES: %*.*s\n", (int)len, (int)len, data); } while (!mysql_session_track_get_next(mysql, SESSION_TRACK_SYSTEM_VARIABLES, &data, &len)); diag("charset: %s", mysql->charset->csname); - FAIL_IF(strcmp(mysql->charset->csname, "utf8"), "Expected charset 'utf8'"); + FAIL_IF(strcmp(mysql->charset->csname, get_utf8_name(mysql_get_server_version(mysql),"utf8")), + "Wrong utf8 characterset for this version");

same here

rc= mysql_query(mysql, "SET NAMES latin1"); check_mysql_rc(rc, mysql); diff --git a/unittest/libmariadb/my_test.h b/unittest/libmariadb/my_test.h index c30d1b6d..a040c3d9 100644 --- a/unittest/libmariadb/my_test.h +++ b/unittest/libmariadb/my_test.h @@ -701,3 +701,23 @@ void run_tests(struct my_tests_st *test) { } }

+static inline const char* get_utf8_name(unsigned long server_version, + const char* name) +{ + const char *csname= server_version >= 100600 ? "utf8mb3" : "utf8"; + char *corrected_name= malloc(19*sizeof(char)); + corrected_name[18]='\0'; + + if (!strchr(name, '_')) + { + strcpy(corrected_name,csname); + corrected_name[strlen(csname)]='\0'; + } + else + { + strcpy(corrected_name,csname); + strcat(corrected_name,"_general_ci"); + corrected_name[strlen(csname)+11]= '\0'; + } + return (const char*)corrected_name; +}

shouldn't be needed Regards, Sergei VP of MariaDB Server Engineering and security@mariadb.org