- developers - lists.mariadb.org

Re: [Maria-developers] 186e8e9: MDEV-5171: Add support for --innodb-optimize-keys to mysqldump.
by Sergei Golubchik 14 Jun '15

14 Jun '15

Hi, Jan! On May 04, Jan Lindström wrote: > revision-id: 186e8e91f3ce797258389f6d8729082f7d3ea164 > parent(s): aa5095627e2619bdad7916d33d1016802a84a9e1 > committer: Jan Lindström > branch nick: 10.0-git > timestamp: 2015-05-04 10:23:50 +0300 > message: > > MDEV-5171: Add support for --innodb-optimize-keys to mysqldump. > > Merged revision 93 from bzr+ssh://bazaar.launchpad.net/+branch/percona-server/5.6/ > authored by Alexey Kopytov. > > This patch expands the applicability of InnoDB fast index creation to > mysqldump, ALTER TABLE and OPTIMIZE TABLE as follows: > > 1. mysqldump has now a new option, --innodb-optimize-keys, which changes > the way InnoDB tables are dumped so that secondary and foreign keys are > created after loading the data thus taking advantage of fast index > creation. > > This part of the patch is an implementation of the feature request > reported as MySQL bug #49120. > > More specifically: > > - KEY, UNIQUE KEY and CONSTRAINT specifications are omitted from CREATE > TABLE corresponding to InnoDB tables. > > - an additional ALTER TABLE is issued after dumping the data to create > the previously omitted keys. > > Delaying foreign key creation does not introduce any additional risks as > mysqldump always prepends its output with SET FOREIGN_KEY_CHECKS=0 anyway. > > 2. When ALTER TABLE requires a table copy, secondary keys are now dropped > and recreated later after copying the data. The following restrictions > apply: > > - only non-unique keys can be involved in this optimization > > - if the table contains foreign keys, or a foreign key is being added as > a part of the current ALTER TABLE statement, the optimization is > disabled for all keys. > > This part of the patch is an implementation of the feature request > reported as MySQL bug #57583. > > 3. As OPTIMIZE TABLE is mapped to ALTER TABLE ... ENGINE=InnoDB for > InnoDB tables, it now also benefits from fast index creation with the > same restrictions as for ALTER TABLE. > diff --git a/client/client_priv.h b/client/client_priv.h > index 67a6e82..1846926 100644 > --- a/client/client_priv.h > +++ b/client/client_priv.h > @@ -94,6 +94,7 @@ enum options_client > OPT_SSL_CRL, OPT_SSL_CRLPATH, > OPT_USE_GTID, > OPT_GALERA_SST_MODE, > + OPT_INNODB_OPTIMIZE_KEYS, /* mysqldump */ Not needed, please, remove > OPT_MAX_CLIENT_OPTION /* should be always the last */ > }; > > diff --git a/client/mysqldump.c b/client/mysqldump.c > index e118198..0970082 100644 > --- a/client/mysqldump.c > +++ b/client/mysqldump.c > @@ -375,6 +386,11 @@ static struct my_option my_long_options[] = > "in dump produced with --dump-slave.", &opt_include_master_host_port, > &opt_include_master_host_port, 0, GET_BOOL, NO_ARG, > 0, 0, 0, 0, 0, 0}, > + {"innodb-optimize-keys", OPT_INNODB_OPTIMIZE_KEYS, Here use 0 instead of OPT_INNODB_OPTIMIZE_KEYS > + "Use InnoDB fast index creation by creating secondary indexes after " > + "dumping the data.", > + &opt_innodb_optimize_keys, &opt_innodb_optimize_keys, 0, GET_BOOL, NO_ARG, > + 0, 0, 0, 0, 0, 0}, > {"insert-ignore", OPT_INSERT_IGNORE, "Insert rows with INSERT IGNORE.", > &opt_ignore, &opt_ignore, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, > 0, 0}, > @@ -2581,6 +2597,359 @@ static inline my_bool general_log_or_slow_log_tables(const char *db, > } > > /* > + Find the first occurrence of a quoted identifier in a given string. Returns > + the pointer to the opening quote, and stores the pointer to the closing quote > + to the memory location pointed to by the 'end' argument, > + > + If no quoted identifiers are found, returns NULL (and the value pointed to by > + 'end' is undefined in this case). > +*/ > + > +static const char *parse_quoted_identifier(const char *str, > + const char **end) > +{ > + const char *from; > + const char *to; > + > + if (!(from= strchr(str, '`'))) > + return NULL; This is wrong, see below. Identifiers ar enot necessarily quoted, so this function should parse both quoted and not quoted identifiers (and renamed to, say, parse_next_identifier) > + > + to= from; > + > + while ((to= strchr(to + 1, '`'))) { > + /* > + Double backticks represent a backtick in identifier, rather than a quote > + character. > + */ > + if (to[1] == '`') > + { > + to++; > + continue; > + } > + > + break; > + } > + > + if (to <= from + 1) > + return NULL; /* Empty identifier */ > + > + *end= to; > + > + return from; > +} > + > +/* > + Parse the specified key definition string and check if the key contains an > + AUTO_INCREMENT column as the first key part. We only check for the first key > + part, because unlike MyISAM, InnoDB does not allow the AUTO_INCREMENT column > + as a secondary key column, i.e. the AUTO_INCREMENT column would not be > + considered indexed for such key specification. > +*/ > +static my_bool contains_autoinc_column(const char *autoinc_column, > + const char *keydef, > + key_type_t type) > +{ > + const char *from, *to; > + uint idnum; > + > + DBUG_ASSERT(type != KEY_TYPE_NONE); > + > + if (autoinc_column == NULL) > + return FALSE; > + > + idnum= 0; > + > + /* > + There is only 1 iteration of the following loop for type == KEY_TYPE_PRIMARY > + and 2 iterations for type == KEY_TYPE_UNIQUE / KEY_TYPE_NON_UNIQUE. > + */ > + while ((from= parse_quoted_identifier(keydef, &to))) > + { > + idnum++; > + > + /* > + Skip the check if it's the first identifier and we are processing a > + secondary key. > + */ > + if ((type == KEY_TYPE_PRIMARY || idnum != 1) && > + !strncmp(autoinc_column, from + 1, to - from - 1)) > + return TRUE; > + > + /* > + Check only the first (for PRIMARY KEY) or the second (for secondary keys) > + quoted identifier. > + */ > + if ((idnum == 1 + MY_TEST(type != KEY_TYPE_PRIMARY))) > + break; > + > + keydef= to + 1; > + } > + > + return FALSE; > +} > + > +/* > + Find a node in the skipped keys list whose name matches a quoted > + identifier specified as 'id_from' and 'id_to' arguments. > +*/ > + > +static LIST *find_matching_skipped_key(const char *id_from, > + const char *id_to) > +{ > + LIST *list; > + size_t id_len; > + > + id_len= id_to - id_from + 1; > + DBUG_ASSERT(id_len > 2); > + > + for (list= skipped_keys_list; list; list= list_rest(list)) > + { > + const char *keydef; > + const char *keyname_from; > + const char *keyname_to; > + size_t keyname_len; > + > + keydef= list->data; > + > + if ((keyname_from= parse_quoted_identifier(keydef, &keyname_to))) > + { > + keyname_len= keyname_to - keyname_from + 1; > + > + if (id_len == keyname_len && > + !strncmp(keyname_from, id_from, id_len)) > + return list; > + } > + } > + > + return NULL; > +} > + > +/* > + Remove secondary/foreign key definitions from a given SHOW CREATE TABLE string > + and store them into a temporary list to be used later. > + > + SYNOPSIS > + skip_secondary_keys() > + create_str SHOW CREATE TABLE output > + has_pk TRUE, if the table has PRIMARY KEY > + (or UNIQUE key on non-nullable columns) > + > + > + DESCRIPTION > + > + Stores all lines starting with "KEY" or "UNIQUE KEY" > + into skipped_keys_list and removes them from the input string. > + Ignoring FOREIGN KEYS constraints when creating the table is ok, because > + mysqldump sets foreign_key_checks to 0 anyway. > +*/ > + > +static void skip_secondary_keys(char *create_str, my_bool has_pk) > +{ > + char *ptr, *strend; > + char *last_comma= NULL; > + my_bool pk_processed= FALSE; > + char *autoinc_column= NULL; > + my_bool has_autoinc= FALSE; > + key_type_t type; > + const char *constr_from; > + const char *constr_to; > + LIST *keydef_node; > + my_bool keys_processed= FALSE; > + > + strend= create_str + strlen(create_str); > + > + ptr= create_str; > + while (*ptr && !keys_processed) > + { please, fix the coding style (here and everywhere in the patch) to follow the rest of the file. That is while { char *tmp, *orig_ptr, c; ... > + char *tmp, *orig_ptr, c; > + > + orig_ptr= ptr; > + /* Skip leading whitespace */ > + while (*ptr && my_isspace(charset_info, *ptr)) > + ptr++; > + > + /* Read the next line */ > + for (tmp= ptr; *tmp != '\n' && *tmp != '\0'; tmp++); > + > + c= *tmp; > + *tmp= '\0'; /* so strstr() only processes the current line */ > + > + if (!strncmp(ptr, "CONSTRAINT ", sizeof("CONSTRAINT ") - 1) && > + (constr_from= parse_quoted_identifier(ptr, &constr_to)) && yuck! identifiers do not have to be quoted. this code needs to be fixed to accept non-quoted identifiers too. > + (keydef_node= find_matching_skipped_key(constr_from, constr_to))) > + { > + char *keydef; > + size_t keydef_len; > + > + /* > + There's a skipped key with the same name as the constraint name. Let's > + put it back before the current constraint definition and remove from the > + skipped keys list. 1. fix the formatting, please 2. why not to create the FK constraint afterwards? > + */ > + keydef= keydef_node->data; > + keydef_len= strlen(keydef) + 5; /* ", \n " */ > + > + memmove(orig_ptr + keydef_len, orig_ptr, strend - orig_ptr + 1); > + memcpy(ptr, keydef, keydef_len - 5); > + memcpy(ptr + keydef_len - 5, ", \n ", 5); > + > + skipped_keys_list= list_delete(skipped_keys_list, keydef_node); > + my_free(keydef); > + my_free(keydef_node); > + > + strend+= keydef_len; > + orig_ptr+= keydef_len; > + ptr+= keydef_len; > + tmp+= keydef_len; > + > + type= KEY_TYPE_NONE; > + } > + else if (!strncmp(ptr, "UNIQUE KEY ", sizeof("UNIQUE KEY ") - 1)) > + type= KEY_TYPE_UNIQUE; > + else if (!strncmp(ptr, "KEY ", sizeof("KEY ") - 1)) > + type= KEY_TYPE_NON_UNIQUE; > + else if (!strncmp(ptr, "PRIMARY KEY ", sizeof("PRIMARY KEY ") - 1)) > + type= KEY_TYPE_PRIMARY; > + else > + type= KEY_TYPE_NONE; > + > + has_autoinc= (type != KEY_TYPE_NONE) ? > + contains_autoinc_column(autoinc_column, ptr, type) : FALSE; > + > + /* Is it a secondary index definition? */ > + if (c == '\n' && > + ((type == KEY_TYPE_UNIQUE && (pk_processed || !has_pk)) || > + type == KEY_TYPE_NON_UNIQUE) && !has_autoinc) > + { > + char *data, *end= tmp - 1; > + > + /* Remove the trailing comma */ > + if (*end == ',') > + end--; > + data= my_strndup(ptr, end - ptr + 1, MYF(MY_FAE)); > + skipped_keys_list= list_cons(data, skipped_keys_list); > + > + memmove(orig_ptr, tmp + 1, strend - tmp); > + ptr= orig_ptr; > + strend-= tmp + 1 - ptr; > + > + /* Remove the comma on the previos line */ > + if (last_comma != NULL) > + { > + *last_comma= ' '; > + } > + } > + else > + { > + char *end; > + > + if (last_comma != NULL && *ptr == ')') > + { > + keys_processed= TRUE; > + } > + else if (last_comma != NULL && !keys_processed) > + { > + /* > + It's not the last line of CREATE TABLE, so we have skipped a key > + definition. We have to restore the last removed comma. > + */ > + *last_comma= ','; > + } > + > + /* > + If we are skipping a key which indexes an AUTO_INCREMENT column, it is > + safe to optimize all subsequent keys, i.e. we should not be checking for > + that column anymore. > + */ > + if (type != KEY_TYPE_NONE && has_autoinc) > + { > + DBUG_ASSERT(autoinc_column != NULL); > + > + my_free(autoinc_column); > + autoinc_column= NULL; > + } > + > + if ((has_pk && type == KEY_TYPE_UNIQUE && !pk_processed) || > + type == KEY_TYPE_PRIMARY) > + pk_processed= TRUE; > + > + if (strstr(ptr, "AUTO_INCREMENT") && *ptr == '`') > + { > + /* > + The first secondary key defined on this column later cannot be > + skipped, as CREATE TABLE would fail on import. Unless there is a > + PRIMARY KEY and it indexes that column. > + */ > + for (end= ptr + 1; > + /* Skip double backticks as they are a part of identifier */ > + *end != '\0' && (*end != '`' || end[1] == '`'); > + end++) > + /* empty */; > + > + if (*end == '`' && end > ptr + 1) > + { > + DBUG_ASSERT(autoinc_column == NULL); > + > + autoinc_column= my_strndup(ptr + 1, end - ptr - 1, MYF(MY_FAE)); Eh? Why does it need to strdup? > + } > + } > + > + *tmp= c; > + > + if (tmp[-1] == ',') > + last_comma= tmp - 1; > + ptr= (*tmp == '\0') ? tmp : tmp + 1; > + } > + } > + > + my_free(autoinc_column); > +} > + > +/* > + Check if the table has a primary key defined either explicitly or > + implicitly (i.e. a unique key on non-nullable columns). > + > + SYNOPSIS > + my_bool has_primary_key(const char *table_name) > + > + table_name quoted table name > + > + RETURNS TRUE if the table has a primary key > + > + DESCRIPTION > +*/ > + > +static my_bool has_primary_key(const char *table_name) > +{ > + MYSQL_RES *res= NULL; > + MYSQL_ROW row; > + char query_buff[QUERY_LENGTH]; > + my_bool has_pk= TRUE; > + > + my_snprintf(query_buff, sizeof(query_buff), > + "SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE " > + "TABLE_SCHEMA=DATABASE() AND TABLE_NAME='%s' AND " > + "COLUMN_KEY='PRI'", table_name); > + if (mysql_query(mysql, query_buff) || !(res= mysql_store_result(mysql)) || > + !(row= mysql_fetch_row(res))) > + { > + fprintf(stderr, "Warning: Couldn't determine if table %s has a " > + "primary key (%s). " > + "--innodb-optimize-keys may work inefficiently.\n", > + table_name, mysql_error(mysql)); > + goto cleanup; > + } > + > + has_pk= atoi(row[0]) > 0; > + > + cleanup: > + if (res) > + mysql_free_result(res); > + > + return has_pk; > +} > + > +/* > get_table_structure -- retrievs database structure, prints out corresponding > CREATE statement and fills out insert_pat if the table is the type we will > be dumping. > @@ -2659,6 +3029,9 @@ static uint get_table_structure(char *table, char *db, char *table_type, > result_table= quote_name(table, table_buff, 1); > opt_quoted_table= quote_name(table, table_buff2, 0); > > + if (opt_innodb_optimize_keys && !strcmp(table_type, "InnoDB")) > + has_pk= has_primary_key(table); > + see below > if (opt_order_by_primary) > order_by= primary_key_fields(result_table); > > @@ -2838,6 +3211,9 @@ static uint get_table_structure(char *table, char *db, char *table_type, > > row= mysql_fetch_row(result); > > + if (opt_innodb_optimize_keys && !strcmp(table_type, "InnoDB")) > + skip_secondary_keys(row[1], has_pk); > + Not that it makes a lot of difference, but I'd remove the previous hunk, the variable has_pk, and put everything here, as if (opt_innodb_optimize_keys && !strcmp(table_type, "InnoDB")) skip_secondary_keys(row[1], has_primary_key(table)); this removes one if(), one strcmp(), and doesn't invoke has_primary_key() (meaning, one SQL query less) unless all other conditions for skip_secondary_keys() are met. even better - don't do has_primary_key() at all, remove the function altogether, and look for "PRIMARY KEY" in the create string inside skip_secondary_keys(). > is_log_table= general_log_or_slow_log_tables(db, table); > if (is_log_table) > row[1]+= 13; /* strlen("CREATE TABLE ")= 13 */ Regards, Sergei

1 0

Re: [Maria-developers] Features Wish-list in MariaDB 10.1.
by Kristian Nielsen 12 Jun '15

12 Jun '15

So my opinion on these below two points was requested, let's keep it on the public mailing list: > * Crash-Safe replication with filename and position (not depending on > GTIDs) > Something similar as what is described in the following link would > be good: > http://blog.booking.com/better_crash_safe_replication_for_mysql.html > * Have a way to avoid "SELECT binlog_gtid_pos" when a slave connect to > the master in MASTER_USE_GTID = no So my personal opinion on this is that I prefer to focus on improving the MariaDB GTID implementation. Considerable effort was invested into this, and the idea is that future development will focus on GTID mode, keeping non-GTID mode only as a backward compatibility. Significant effort was made to ensure that MariaDB GTID is usable for all existing use-cases of replication. For the first point, GTID replication is already crash-safe. For the second point, for GTID we in any case need a solution for locating GTID position quickly, otherwise GTID replication can have performance problems in case of frequent slave connects (and Jonas Oreland has promised a patch for this any day now ;-). And once this solution is in, the second point should be moot anyway. Of course, others may have a different opinion. For the second case, I suppose the easiest thing is just to remove the code that does SELECT binlog_gtid_pos() from the slave, if no-one cares about that functionality anyway... Hope this helps, - Kristian.

2 2

[Maria-developers] 10.0 sprint: Please advise re MDEV-8109 unexpected CAST result
by Alexander Barkov 11 Jun '15

11 Jun '15

Hi Sergei, I'm thinking about what to do with MDEV-8109 unexpected CAST result. Observations: 1. An empty string in VALUES SET sql_mode='STRICT_ALL_TABLES'; DROP TABLE IF EXISTS t1; CREATE TABLE t1 (a DECIMAL); INSERT INTO t1 VALUES (''); ERROR 1292 (22007): Truncated incorrect DECIMAL value: '' This is OK. In strict mode INSERT with a bad value correctly returns the error. 2. CAST('' AS DECIMAL) SET sql_mode='STRICT_ALL_TABLES'; SELECT CAST('' AS DECIMAL); +---------------------+ | CAST('' AS DECIMAL) | +---------------------+ | 0 | +---------------------+ 1 row in set, 1 warning (0.00 sec) This is also OK. There are no tables involved, and CAST itself does not convert warnings to errors. 3. CAST('' AS DECIMAL) in VALUES: INSERT INTO t1 VALUES(CAST('' AS DECIMAL)); ERROR 1292 (22007): Truncated incorrect DECIMAL value: '' I think this is not OK. I wrote an explicit CAST, so I expect: - CAST to return 0 with a warning, like in #2 - INSERT to write 0 into the table normally, as I'm actually inserting the CAST result, which is 0. 4. COLUMN_GET(AS DECIMAL) in VALUES: SET @aaa = COLUMN_CREATE('price', ''); INSERT INTO t1 VALUES(COLUMN_GET(@aaa, 'price' AS DECIMAL)); ERROR 1918 (22007): Encountered illegal value '' when converting to DECIMAL This is also not OK. I wrote explicit column conversion to DECIMAL. I expect: - COLUMN_GET() to return 0 with a warning, like in #2 - INSERT to write 0 into the table normally, as I'm actually inserting the COLUMN_GET result, which is 0. Do you agree? Thanks.

2 4

Re: [Maria-developers] MariaDB replication event generators?
by Kristian Nielsen 11 Jun '15

11 Jun '15

James Cheng <jcheng(a)tivo.com> writes: > The binlog stream has all the row data, but doesn't have > everything. It doesn't have column names, for example, or primary > keys, or character sets. Using the binlog for replication seems to > assume that you have a copy of the schema beforehand on the receiving > slave. It doesn't seem possible to replicate to a slave using only the > information in the binlogs. It is a common request to be able to do 3rd-party replication from mysqld. However, the replication protocol between master and slave servers was never designed as a public API, it is based on internal data formats and so on. So unfortunately, there does not exist currently a public API for doing external replication, at least not that I am aware of. The binlog stream _does_ have everything of course, eg. in the form of CREATE TABLE statements and so on. However, being internal to the server, they are not very easy to use by an external replication mechanism. - Kristian.

1 0

Re: [Maria-developers] [Commits] 6ff2c0a: MDEV-5309 - RENAME TABLE does not check for existence of the table's engine
by Sergei Golubchik 10 Jun '15

10 Jun '15

Hi, Sergey! On Feb 09, svoj(a)mariadb.org wrote: > revision-id: 6ff2c0a05155f985484e23bd8611eb95f045907f > parent(s): f13939061d468a47985dee0268652a6fe2db7862 > committer: Sergey Vojtovich > branch nick: 10.1 > timestamp: 2015-02-09 16:12:00 +0400 > message: > > MDEV-5309 - RENAME TABLE does not check for existence of the table's engine > > When RENAME TABLE is executed, it apparently does not check whether the engine > is available (unlike ALTER TABLE .. RENAME, which does). It means that if the > engine in question was not loaded on some reason, the table might become > unusable, since the engine won't know about the change. > > With this patch RENAME TABLE fails if storage engine is not available. .... > diff --git a/sql/sql_table.cc b/sql/sql_table.cc > index e9a1ae9..f1096da 100644 > --- a/sql/sql_table.cc > +++ b/sql/sql_table.cc > @@ -5172,18 +5172,19 @@ bool mysql_create_table(THD *thd, TABLE_LIST *create_table, > error= my_errno; > (void) file->ha_create_partitioning_metadata(to, from, CHF_RENAME_FLAG); > } > - else if (!file || !(error=file->ha_rename_table(from_base, to_base))) > + else if (!file) > { > + error= ER_UNKNOWN_STORAGE_ENGINE; > + } > + else if (!(error= file->ha_rename_table(from_base, to_base))) > + { > if (!(flags & NO_FRM_RENAME) && rename_file_ext(from,to,reg_ext)) > { > error= my_errno; > - if (file) > - { > if (error == ENOENT) > error= 0; // this is ok if file->ha_rename_table() succeeded > else > file->ha_rename_table(to_base, from_base); // Restore old file name > - } > } > } > delete file; May be you'd rather fix it in the caller? Let's state that the first argument of mysql_rename_table() can never be NULL. While mysql_rename_table() is called in many places, the only place where it can actually get a NULL hton is sql_rename.cc, if ha_table_exists() returns hton=NULL. I'd add the check here and simplified mysql_rename_table(). Regards, Sergei

1 0

[Maria-developers] Progress Report on [MDEV5199]: UDF Table Functions.
by Blessing Atie 10 Jun '15

10 Jun '15

Hi Serg, Colin! The following is a rundown on my progress with the above task. I started this project by getting the source of 10.1 from Github, setting my working environment by building and running the tests on ubuntu 14 then proceeded with an analysis by studying plugins and how they interact with the server.(Especially storage engines plugins). With analysis and sources investigations being done, I realized with help from assigned mentor(Serg) that our task is about implementing MYSQL_UDF_PLUGIN with the functionalities of the CONNECT engine which is similar in functionality to table functions in a way. This past week, I worked on the interface for our plugin by declaring a structure with version and some useful init/deinit function pointers to work with our UDFs(table UDF only for now). I am currently studying the Connect engine to see how I can implement its functionalities as a UDF plugin. For this week, I intend to look at other databases for example table functions that will be good candidates that we can implement with our plugin(This was not mentioned in my proposal but I intend to implement a couple of them using our plugin even if we are short of time after the final evaluation). So far, I have had difficulties in communication due to unscheduled powerouts in my city. This has really been an issue to me because my mentor was beginning to loose hope and on the other side, I don't have the chance to ask questions when im faced with a powerout. I really hope my mentors understand with me how difficult it is to code from Cameroon. Please feel free to request for any important details i might have not mentioned which you will want to know. Best Wishes, Blessing Atie

2 1

Re: [Maria-developers] MariaDB replication event generators?
by Kristian Nielsen 10 Jun '15

10 Jun '15

James Cheng <jcheng(a)tivo.com> writes: > On Jun 7, 2015, at 12:05 AM, Kristian Nielsen <knielsen(a)knielsen-hq.org> wrote: > >> James Cheng <jcheng(a)tivo.com> writes: >> >>> I recently read your old blog post from 2010 >>> (http://kristiannielsen.livejournal.com/14305.html) about some ideas >>> for MariaDB replication. In particular, your ideas for replication >>> event generators that would allow plugins to be written to receive >>> replication events. >>> >>> Did this work ever happen in MariaDB? Or anything close/equivalent? If >>> so, could you point me to something so that I can learn more about it? >> >> Unfortunately, nothing like this ever happened. > That's too bad. Are there any interesting details about why it never > happened? Was it simply due to not enough time, not high enough > priority? Was there any technical or organizational resistance to the > actual ideas or functionality? If someone were to build this today and > contribute it back to the community, do you think it would be > accepted? I think there was not enough interest and/or time/priority. For example, Galera cluster would be a prime example for something using this kind of API. But the developers did not prioritise spending time working on such API, rather they hacked in their functionality all over the server. It is the same with other parties in the community, everyone is focused on solving their own narrow problem, not in working together. And MySQL@Oracle, of course, is not interested in anything that involves working with anyone else on the code. So I don't see much chance in the current state of the community of any significant effort being spent neither on developing such an API, nor on using it. - Kristian.

1 0

Re: [Maria-developers] [Commits] 865bbbf: MDEV-3870 - Valgrind warnings on OPTIMIZE MyISAM or Aria TABLE with disabled
by Sergei Golubchik 09 Jun '15

09 Jun '15

Hi, Sergey! On Jun 09, svoj(a)mariadb.org wrote: > revision-id: 865bbbf14d1a8ed500eeeadabe0a7fdfa90420df > parent(s): b8e26b04d4ac22e29a78bca68e101176b6ef507a > committer: Sergey Vojtovich > branch nick: mariadb > timestamp: 2015-06-09 16:08:09 +0400 > message: > > MDEV-3870 - Valgrind warnings on OPTIMIZE MyISAM or Aria TABLE with disabled > keys > > Fixed that OPTIMIZE TABLE against MyISAM/Aria table may write uninitialized key > root position for disabled keys. ok to push, thanks! Regards, Sergei

1 0

Re: [Maria-developers] [Commits] 7948cc3: MDEV-8260 : Issues related to concurrent CTAS
by Sergei Golubchik 09 Jun '15

09 Jun '15

Hi, Nirbhay! On Jun 08, Nirbhay Choubey wrote: > revision-id: 7948cc32b6c12c9696e8b497ae90f123cafcd234 > parent(s): 6d5b723bdc3e04978619b9673fca266e0426916f > committer: Nirbhay Choubey > branch nick: 10.0-galera.ctas > timestamp: 2015-06-08 22:50:26 -0400 > message: > > MDEV-8260 : Issues related to concurrent CTAS > > * Wait for aborted thd (victim) to release MDL locks > * Skip aborting an already aborted thd > * Defer setting OK status in case of CTAS > * Minor cosmetic changes > * Added a test case ... > +bool select_insert::send_eof() > +{ > + DBUG_ENTER("select_insert::send_eof"); > + DBUG_RETURN(prepare_eof() || send_ok_packet()); > } Please, don't do that, don't put function calls in DBUG_RETURN. This will result in dbug traces like > select_insert::send_eof < select_insert::send_eof > select_insert::prepare_eof < select_insert::prepare_eof > select_insert::send_ok_packet < select_insert::send_ok_packet while the correct trace should've be > select_insert::send_eof | > select_insert::prepare_eof | < select_insert::prepare_eof | > select_insert::send_ok_packet | < select_insert::send_ok_packet < select_insert::send_eof So, either write like bool res= prepare_eof() || send_ok_packet(); DBUG_RETURN(res); or fix DBUG_RETURN() macro. Whatever you prefer :) Otherwise the patch is ok. Regards, Sergei

2 1

[Maria-developers] Review requested for MDEV-7818
by Kristian Nielsen 09 Jun '15

09 Jun '15

I need someone to review my below fix for MDEV-7818. This involves the FLUSH TABLES WITH READ LOCK command and associated metadata locking, something that I know little about. Monty suggested the basic semantics - that all currently running parallel replication transactions would be allowed to complete before granting the global read lock. But it really needs a review of the actual implementation, by someone who has an understanding of the FTWRL semantics and implementation. Thanks, - Kristian. Kristian Nielsen <knielsen(a)knielsen-hq.org> writes: > revision-id: 0e609daea4796fafca94f7adf796addfc506168b > parent(s): 22b6c297dc0de6b176238ae820bddb5e70f2c8ab > committer: Kristian Nielsen > branch nick: mariadb > timestamp: 2015-06-09 16:01:53 +0200 > message: > > MDEV-7818: Deadlock occurring with parallel replication and FTWRL > > Problem is that FLUSH TABLES WITH READ LOCK first blocks threads from > starting new commits, then waits for running commits to complete. But > in-order parallel replication needs commits to happen in a particular > order, so this can easily deadlock. > > To fix this problem, this patch introduces a way to temporarily pause > the parallel replication worker threads. Before starting FTWRL, we let > all worker threads complete in-progress transactions, and then > wait. Then we proceed to take the global read lock. Once the lock is > obtained, we unpause the worker threads. Now commits are blocked from > starting by the global read lock, so the deadlock will no longer occur. > > --- > sql/mysqld.cc | 3 + > sql/mysqld.h | 3 + > sql/rpl_parallel.cc | 230 ++++++++++++++++++++++++++++++++++++++++++++++++++-- > sql/rpl_parallel.h | 27 ++++-- > sql/sql_parse.cc | 13 +++ > 5 files changed, 262 insertions(+), 14 deletions(-) > > diff --git a/sql/mysqld.cc b/sql/mysqld.cc > index e05c0b6..af8ae2c 100644 > --- a/sql/mysqld.cc > +++ b/sql/mysqld.cc > @@ -9514,6 +9514,9 @@ PSI_stage_info stage_waiting_for_prior_transaction_to_commit= { 0, "Waiting for > PSI_stage_info stage_waiting_for_prior_transaction_to_start_commit= { 0, "Waiting for prior transaction to start commit before starting next transaction", 0}; > PSI_stage_info stage_waiting_for_room_in_worker_thread= { 0, "Waiting for room in worker thread event queue", 0}; > PSI_stage_info stage_waiting_for_workers_idle= { 0, "Waiting for worker threads to be idle", 0}; > +PSI_stage_info stage_waiting_for_ftwrl= { 0, "Waiting due to global read lock", 0}; > +PSI_stage_info stage_waiting_for_ftwrl_threads_to_pause= { 0, "Waiting for worker threads to pause for global read lock", 0}; > +PSI_stage_info stage_waiting_for_rpl_thread_pool= { 0, "Waiting while replication worker thread pool is busy", 0}; > PSI_stage_info stage_master_gtid_wait_primary= { 0, "Waiting in MASTER_GTID_WAIT() (primary waiter)", 0}; > PSI_stage_info stage_master_gtid_wait= { 0, "Waiting in MASTER_GTID_WAIT()", 0}; > PSI_stage_info stage_gtid_wait_other_connection= { 0, "Waiting for other master connection to process GTID received on multiple master connections", 0}; > diff --git a/sql/mysqld.h b/sql/mysqld.h > index 156e7f9..8c953c1 100644 > --- a/sql/mysqld.h > +++ b/sql/mysqld.h > @@ -454,6 +454,9 @@ extern PSI_stage_info stage_waiting_for_prior_transaction_to_commit; > extern PSI_stage_info stage_waiting_for_prior_transaction_to_start_commit; > extern PSI_stage_info stage_waiting_for_room_in_worker_thread; > extern PSI_stage_info stage_waiting_for_workers_idle; > +extern PSI_stage_info stage_waiting_for_ftwrl; > +extern PSI_stage_info stage_waiting_for_ftwrl_threads_to_pause; > +extern PSI_stage_info stage_waiting_for_rpl_thread_pool; > extern PSI_stage_info stage_master_gtid_wait_primary; > extern PSI_stage_info stage_master_gtid_wait; > extern PSI_stage_info stage_gtid_wait_other_connection; > diff --git a/sql/rpl_parallel.cc b/sql/rpl_parallel.cc > index 738d0e5..d90507a 100644 > --- a/sql/rpl_parallel.cc > +++ b/sql/rpl_parallel.cc > @@ -280,6 +280,8 @@ do_gco_wait(rpl_group_info *rgi, group_commit_orderer *gco, > rpl_parallel_entry *entry= rgi->parallel_entry; > uint64 wait_count; > > + mysql_mutex_assert_owner(&entry->LOCK_parallel_entry); > + > if (!gco->installed) > { > group_commit_orderer *prev_gco= gco->prev_gco; > @@ -336,6 +338,159 @@ do_gco_wait(rpl_group_info *rgi, group_commit_orderer *gco, > } > > > +static void > +do_ftwrl_wait(rpl_group_info *rgi, > + bool *did_enter_cond, PSI_stage_info *old_stage) > +{ > + THD *thd= rgi->thd; > + rpl_parallel_entry *entry= rgi->parallel_entry; > + uint64 sub_id= rgi->gtid_sub_id; > + > + mysql_mutex_assert_owner(&entry->LOCK_parallel_entry); > + > + if (unlikely(entry->pause_sub_id > 0) && sub_id > entry->pause_sub_id) > + { > + thd->ENTER_COND(&entry->COND_parallel_entry, &entry->LOCK_parallel_entry, > + &stage_waiting_for_ftwrl, old_stage); > + *did_enter_cond= true; > + do > + { > + if (entry->force_abort || rgi->worker_error) > + break; > + if (thd->check_killed()) > + { > + thd->send_kill_message(); > + slave_output_error_info(rgi, thd); > + signal_error_to_sql_driver_thread(thd, rgi, 1); > + break; > + } > + mysql_cond_wait(&entry->COND_parallel_entry, &entry->LOCK_parallel_entry); > + } while (entry->pause_sub_id > 0 && sub_id > entry->pause_sub_id); > + } > + > + if (sub_id > entry->largest_started_sub_id) > + entry->largest_started_sub_id= sub_id; > +} > + > + > +static void > +pool_mark_busy(rpl_parallel_thread_pool *pool) > +{ > + mysql_mutex_assert_owner(&pool->LOCK_rpl_thread_pool); > + DBUG_ASSERT(!pool->busy); > + pool->busy= true; > +} > + > + > +static void > +pool_mark_not_busy(rpl_parallel_thread_pool *pool) > +{ > + mysql_mutex_assert_owner(&pool->LOCK_rpl_thread_pool); > + DBUG_ASSERT(pool->busy); > + pool->busy= false; > + mysql_cond_broadcast(&pool->COND_rpl_thread_pool); > +} > + > + > +void > +rpl_unpause_for_ftwrl(THD *thd) > +{ > + uint32 i; > + rpl_parallel_thread_pool *pool= &global_rpl_thread_pool; > + > + DBUG_ASSERT(pool->busy); > + > + for (i= 0; i < pool->count; ++i) > + { > + rpl_parallel_entry *e; > + rpl_parallel_thread *rpt= pool->threads[i]; > + > + mysql_mutex_lock(&rpt->LOCK_rpl_thread); > + if (!rpt->current_owner) > + { > + mysql_mutex_unlock(&rpt->LOCK_rpl_thread); > + continue; > + } > + e= rpt->current_entry; > + mysql_mutex_lock(&e->LOCK_parallel_entry); > + mysql_mutex_unlock(&rpt->LOCK_rpl_thread); > + e->pause_sub_id= 0; > + mysql_cond_broadcast(&e->COND_parallel_entry); > + } > + > + mysql_mutex_lock(&pool->LOCK_rpl_thread_pool); > + pool_mark_not_busy(pool); > + mysql_cond_broadcast(&pool->COND_rpl_thread_pool); > + mysql_mutex_unlock(&pool->LOCK_rpl_thread_pool); > +} > + > + > +/* > + . > + > + Note: in case of error return, unpause_for_ftwrl() must _not_ be called. > +*/ > +int > +rpl_pause_for_ftwrl(THD *thd) > +{ > + uint32 i; > + rpl_parallel_thread_pool *pool= &global_rpl_thread_pool; > + int err= 0; > + > + /* > + While the count_pending_pause_for_ftwrl counter is non-zero, the pool > + cannot be shutdown/resized, so threads are guaranteed to not disappear. > + > + This is required to safely be able to access the individual threads below. > + (We cannot lock an individual thread while holding LOCK_rpl_thread_pool, > + as this can deadlock against release_thread()). > + */ > + mysql_mutex_lock(&pool->LOCK_rpl_thread_pool); > + pool_mark_busy(pool); > + mysql_mutex_unlock(&pool->LOCK_rpl_thread_pool); > + > + for (i= 0; i < pool->count; ++i) > + { > + PSI_stage_info old_stage; > + rpl_parallel_entry *e; > + rpl_parallel_thread *rpt= pool->threads[i]; > + > + mysql_mutex_lock(&rpt->LOCK_rpl_thread); > + if (!rpt->current_owner) > + { > + mysql_mutex_unlock(&rpt->LOCK_rpl_thread); > + continue; > + } > + e= rpt->current_entry; > + mysql_mutex_lock(&e->LOCK_parallel_entry); > + mysql_mutex_unlock(&rpt->LOCK_rpl_thread); > + ++e->need_sub_id_signal; > + if (!e->pause_sub_id) > + e->pause_sub_id= e->largest_started_sub_id; > + thd->ENTER_COND(&e->COND_parallel_entry, &e->LOCK_parallel_entry, > + &stage_waiting_for_ftwrl_threads_to_pause, &old_stage); > + while (e->last_committed_sub_id < e->pause_sub_id && !err) > + { > + if (thd->check_killed()) > + { > + thd->send_kill_message(); > + err= 1; > + break; > + } > + mysql_cond_wait(&e->COND_parallel_entry, &e->LOCK_parallel_entry); > + }; > + --e->need_sub_id_signal; > + thd->EXIT_COND(&old_stage); > + if (err) > + break; > + } > + > + if (err) > + rpl_unpause_for_ftwrl(thd); > + return err; > +} > + > + > #ifndef DBUG_OFF > static int > dbug_simulate_tmp_error(rpl_group_info *rgi, THD *thd) > @@ -856,6 +1011,9 @@ handle_rpl_parallel_thread(void *arg) > > if (unlikely(entry->stop_on_error_sub_id <= rgi->wait_commit_sub_id)) > skip_event_group= true; > + if (likely(!skip_event_group)) > + do_ftwrl_wait(rgi, &did_enter_cond, &old_stage); > + > register_wait_for_prior_event_group_commit(rgi, entry); > > unlock_or_exit_cond(thd, &entry->LOCK_parallel_entry, > @@ -1058,6 +1216,47 @@ rpl_parallel_change_thread_count(rpl_parallel_thread_pool *pool, > rpl_parallel_thread **new_list= NULL; > rpl_parallel_thread *new_free_list= NULL; > rpl_parallel_thread *rpt_array= NULL; > + THD *thd; > + PSI_stage_info old_stage; > + int res; > + > + /* > + Wait here while the queue is busy. This is done to make FLUSH TABLES WITH > + READ LOCK work correctly, without incuring extra locking penalties in > + normal operation. FLUSH TABLES WITH READ LOCK needs to lock threads in the > + thread pool, and for this we need to make sure the pool will not go away > + during the operation. The LOCK_rpl_thread_pool is not suitable for > + this. It is taken by release_thread() while holding LOCK_rpl_thread; so it > + must be released before locking any LOCK_rpl_thread lock, or a deadlock > + can occur. > + > + So we protect the infrequent operations of FLUSH TABLES WITH READ LOCK and > + pool size changes with this condition wait. > + */ > + thd= current_thd; > + res= 0; > + mysql_mutex_lock(&pool->LOCK_rpl_thread_pool); > + if (thd) > + thd->ENTER_COND(&pool->COND_rpl_thread_pool, &pool->LOCK_rpl_thread_pool, > + &stage_waiting_for_rpl_thread_pool, &old_stage); > + while (pool->busy) > + { > + if (thd->check_killed()) > + { > + thd->send_kill_message(); > + res= 1; > + break; > + } > + mysql_cond_wait(&pool->COND_rpl_thread_pool, &pool->LOCK_rpl_thread_pool); > + } > + if (!res) > + pool_mark_busy(pool); > + if (thd) > + thd->EXIT_COND(&old_stage); > + else > + mysql_mutex_unlock(&pool->LOCK_rpl_thread_pool); > + if (res) > + return res; > > /* > Allocate the new list of threads up-front. > @@ -1106,7 +1305,14 @@ rpl_parallel_change_thread_count(rpl_parallel_thread_pool *pool, > */ > for (i= 0; i < pool->count; ++i) > { > - rpl_parallel_thread *rpt= pool->get_thread(NULL, NULL); > + rpl_parallel_thread *rpt; > + > + mysql_mutex_lock(&pool->LOCK_rpl_thread_pool); > + while ((rpt= pool->free_list) == NULL) > + mysql_cond_wait(&pool->COND_rpl_thread_pool, &pool->LOCK_rpl_thread_pool); > + pool->free_list= rpt->next; > + mysql_mutex_unlock(&pool->LOCK_rpl_thread_pool); > + mysql_mutex_lock(&rpt->LOCK_rpl_thread); > rpt->stop= true; > mysql_cond_signal(&rpt->COND_rpl_thread); > mysql_mutex_unlock(&rpt->LOCK_rpl_thread); > @@ -1157,7 +1363,7 @@ rpl_parallel_change_thread_count(rpl_parallel_thread_pool *pool, > } > > mysql_mutex_lock(&pool->LOCK_rpl_thread_pool); > - mysql_cond_broadcast(&pool->COND_rpl_thread_pool); > + pool_mark_not_busy(pool); > mysql_mutex_unlock(&pool->LOCK_rpl_thread_pool); > > return 0; > @@ -1182,6 +1388,9 @@ rpl_parallel_change_thread_count(rpl_parallel_thread_pool *pool, > } > my_free(new_list); > } > + mysql_mutex_lock(&pool->LOCK_rpl_thread_pool); > + pool_mark_not_busy(pool); > + mysql_mutex_unlock(&pool->LOCK_rpl_thread_pool); > return 1; > } > > @@ -1445,7 +1654,7 @@ rpl_parallel_thread::loc_free_gco(group_commit_orderer *gco) > > > rpl_parallel_thread_pool::rpl_parallel_thread_pool() > - : count(0), threads(0), free_list(0), inited(false) > + : threads(0), free_list(0), count(0), inited(false), busy(false) > { > } > > @@ -1453,9 +1662,10 @@ rpl_parallel_thread_pool::rpl_parallel_thread_pool() > int > rpl_parallel_thread_pool::init(uint32 size) > { > - count= 0; > threads= NULL; > free_list= NULL; > + count= 0; > + busy= false; > > mysql_mutex_init(key_LOCK_rpl_thread_pool, &LOCK_rpl_thread_pool, > MY_MUTEX_INIT_SLOW); > @@ -1496,8 +1706,14 @@ rpl_parallel_thread_pool::get_thread(rpl_parallel_thread **owner, > rpl_parallel_thread *rpt; > > mysql_mutex_lock(&LOCK_rpl_thread_pool); > - while ((rpt= free_list) == NULL) > + for (;;) > + { > + while (unlikely(busy)) > + mysql_cond_wait(&COND_rpl_thread_pool, &LOCK_rpl_thread_pool); > + if ((rpt= free_list) != NULL) > + break; > mysql_cond_wait(&COND_rpl_thread_pool, &LOCK_rpl_thread_pool); > + } > free_list= rpt->next; > mysql_mutex_unlock(&LOCK_rpl_thread_pool); > mysql_mutex_lock(&rpt->LOCK_rpl_thread); > @@ -1908,7 +2124,7 @@ rpl_parallel::wait_for_workers_idle(THD *thd) > > e= (struct rpl_parallel_entry *)my_hash_element(&domain_hash, i); > mysql_mutex_lock(&e->LOCK_parallel_entry); > - e->need_sub_id_signal= true; > + ++e->need_sub_id_signal; > thd->ENTER_COND(&e->COND_parallel_entry, &e->LOCK_parallel_entry, > &stage_waiting_for_workers_idle, &old_stage); > while (e->current_sub_id > e->last_committed_sub_id) > @@ -1921,7 +2137,7 @@ rpl_parallel::wait_for_workers_idle(THD *thd) > } > mysql_cond_wait(&e->COND_parallel_entry, &e->LOCK_parallel_entry); > } > - e->need_sub_id_signal= false; > + --e->need_sub_id_signal; > thd->EXIT_COND(&old_stage); > if (err) > return err; > diff --git a/sql/rpl_parallel.h b/sql/rpl_parallel.h > index 04f8706..77a4005 100644 > --- a/sql/rpl_parallel.h > +++ b/sql/rpl_parallel.h > @@ -199,12 +199,18 @@ struct rpl_parallel_thread { > > > struct rpl_parallel_thread_pool { > - uint32 count; > struct rpl_parallel_thread **threads; > struct rpl_parallel_thread *free_list; > mysql_mutex_t LOCK_rpl_thread_pool; > mysql_cond_t COND_rpl_thread_pool; > + uint32 count; > bool inited; > + /* > + While FTWRL runs, this counter is incremented to make SQL thread or > + STOP/START slave not try to start new activity while that operation > + is in progress. > + */ > + bool busy; > > rpl_parallel_thread_pool(); > int init(uint32 size); > @@ -219,6 +225,12 @@ struct rpl_parallel_entry { > mysql_mutex_t LOCK_parallel_entry; > mysql_cond_t COND_parallel_entry; > uint32 domain_id; > + /* > + Incremented by wait_for_workers_idle() and rpl_pause_for_ftwrl() to show > + that they are waiting, so that finish_event_group knows to signal them > + when last_committed_sub_id is increased. > + */ > + uint32 need_sub_id_signal; > uint64 last_commit_id; > bool active; > /* > @@ -228,12 +240,6 @@ struct rpl_parallel_entry { > */ > bool force_abort; > /* > - Set in wait_for_workers_idle() to show that it is waiting, so that > - finish_event_group knows to signal it when last_committed_sub_id is > - increased. > - */ > - bool need_sub_id_signal; > - /* > At STOP SLAVE (force_abort=true), we do not want to process all events in > the queue (which could unnecessarily delay stop, if a lot of events happen > to be queued). The stop_count provides a safe point at which to stop, so > @@ -291,6 +297,11 @@ struct rpl_parallel_entry { > The value is ULONGLONG_MAX when no error occured. > */ > uint64 stop_on_error_sub_id; > + /* > + During FLUSH TABLES WITH READ LOCK, transactions with sub_id larger than > + this value must not start, but wait until the global read lock is released. > + */ > + uint64 pause_sub_id; > /* Total count of event groups queued so far. */ > uint64 count_queued_event_groups; > /* > @@ -331,5 +342,7 @@ extern struct rpl_parallel_thread_pool global_rpl_thread_pool; > extern int rpl_parallel_activate_pool(rpl_parallel_thread_pool *pool); > extern int rpl_parallel_inactivate_pool(rpl_parallel_thread_pool *pool); > extern bool process_gtid_for_restart_pos(Relay_log_info *rli, rpl_gtid *gtid); > +extern int rpl_pause_for_ftwrl(THD *thd); > +extern void rpl_unpause_for_ftwrl(THD *thd); > > #endif /* RPL_PARALLEL_H */ > diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc > index 5635e9a..ca6b11e 100644 > --- a/sql/sql_parse.cc > +++ b/sql/sql_parse.cc > @@ -4271,6 +4271,17 @@ case SQLCOM_PREPARE: > break; > } > > + if (lex->type & REFRESH_READ_LOCK) > + { > + /* > + We need to pause any parallel replication slave workers during FLUSH > + TABLES WITH READ LOCK. Otherwise we might cause a deadlock, as > + worker threads eun run in arbitrary order but need to commit in a > + specific given order. > + */ > + if (rpl_pause_for_ftwrl(thd)) > + goto error; > + } > /* > reload_acl_and_cache() will tell us if we are allowed to write to the > binlog or not. > @@ -4301,6 +4312,8 @@ case SQLCOM_PREPARE: > if (!res) > my_ok(thd); > } > + if (lex->type & REFRESH_READ_LOCK) > + rpl_unpause_for_ftwrl(thd); > > break; > } > _______________________________________________ > commits mailing list > commits(a)mariadb.org > https://lists.askmonty.org/cgi-bin/mailman/listinfo/commits

1 0