- developers - lists.mariadb.org

[Maria-developers] Rev 2731: Remove percona's mysql-test/patches from MariaDB (we want to get a visible in file:///home/psergey/bzr-new/mysql-5.1-maria-contd3/
by Sergey Petrunya 15 Sep '09

15 Sep '09

At file:///home/psergey/bzr-new/mysql-5.1-maria-contd3/ ------------------------------------------------------------ revno: 2731 revision-id: psergey(a)askmonty.org-20090914225219-dlevkjuu61j32vge parent: psergey(a)askmonty.org-20090914224548-yho0nxw78vvxd5yi committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: mysql-5.1-maria-contd3 timestamp: Tue 2009-09-15 02:52:19 +0400 message: Remove percona's mysql-test/patches from MariaDB (we want to get a visible conflict every time they make a change) === removed directory 'mysql-test/patches' === removed file 'mysql-test/patches/events_stress.diff' --- a/mysql-test/patches/events_stress.diff 2009-09-08 16:04:58 +0000 +++ b/mysql-test/patches/events_stress.diff 1970-01-01 00:00:00 +0000 @@ -1,33 +0,0 @@ ---- mysql-test/t/events_stress.test.orig 2009-07-05 10:29:14.000000000 +0000 -+++ mysql-test/t/events_stress.test 2009-07-05 10:30:49.000000000 +0000 -@@ -61,6 +61,7 @@ - } - --enable_query_log - SELECT COUNT(*) FROM INFORMATION_SCHEMA.EVENTS WHERE EVENT_SCHEMA='events_conn1_test2'; -+SET @old_event_scheduler=@@event_scheduler; - SET GLOBAL event_scheduler=on; - --sleep 2.5 - DROP DATABASE events_conn1_test2; -@@ -137,5 +138,5 @@ - DROP DATABASE events_test; - - # Cleanup --SET GLOBAL event_scheduler=off; -+SET GLOBAL event_scheduler=@old_event_scheduler; - --source include/check_events_off.inc ---- mysql-test/r/events_stress.result.orig 2009-07-05 10:54:30.000000000 +0000 -+++ mysql-test/r/events_stress.result 2009-07-05 10:54:48.000000000 +0000 -@@ -32,6 +32,7 @@ - SELECT COUNT(*) FROM INFORMATION_SCHEMA.EVENTS WHERE EVENT_SCHEMA='events_conn1_test2'; - COUNT(*) - 50 -+SET @old_event_scheduler=@@event_scheduler; - SET GLOBAL event_scheduler=on; - DROP DATABASE events_conn1_test2; - SET GLOBAL event_scheduler=off; -@@ -63,4 +64,4 @@ - DROP TABLE fill_it2; - DROP TABLE fill_it3; - DROP DATABASE events_test; --SET GLOBAL event_scheduler=off; -+SET GLOBAL event_scheduler=@old_event_scheduler; === removed file 'mysql-test/patches/information_schema.diff' --- a/mysql-test/patches/information_schema.diff 2009-09-08 16:04:58 +0000 +++ b/mysql-test/patches/information_schema.diff 1970-01-01 00:00:00 +0000 @@ -1,129 +0,0 @@ ---- mysql-test/r/information_schema.result.orig 2009-06-25 21:33:28.000000000 +0000 -+++ mysql-test/r/information_schema.result 2009-06-25 21:33:49.000000000 +0000 -@@ -71,6 +71,20 @@ - TRIGGERS - USER_PRIVILEGES - VIEWS -+INNODB_BUFFER_POOL_PAGES_INDEX -+INNODB_RSEG -+INNODB_LOCKS -+INNODB_BUFFER_POOL_PAGES -+XTRADB_ENHANCEMENTS -+INNODB_TRX -+INNODB_INDEX_STATS -+INNODB_LOCK_WAITS -+INNODB_CMP_RESET -+INNODB_CMP -+INNODB_CMPMEM_RESET -+INNODB_BUFFER_POOL_PAGES_BLOB -+INNODB_CMPMEM -+INNODB_TABLE_STATS - columns_priv - db - event -@@ -799,6 +813,8 @@ - TABLES UPDATE_TIME datetime - TABLES CHECK_TIME datetime - TRIGGERS CREATED datetime -+INNODB_TRX trx_started datetime -+INNODB_TRX trx_wait_started datetime - event execute_at datetime - event last_executed datetime - event starts datetime -@@ -847,12 +863,15 @@ - TABLE_CONSTRAINTS TABLE_NAME select - TABLE_PRIVILEGES TABLE_NAME select - VIEWS TABLE_NAME select -+INNODB_BUFFER_POOL_PAGES_INDEX table_name select -+INNODB_INDEX_STATS table_name select -+INNODB_TABLE_STATS table_name select - delete from mysql.user where user='mysqltest_4'; - delete from mysql.db where user='mysqltest_4'; - flush privileges; - SELECT table_schema, count(*) FROM information_schema.TABLES WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test', 'mysqltest') AND table_name<>'ndb_binlog_index' AND table_name<>'ndb_apply_status' GROUP BY TABLE_SCHEMA; - table_schema count(*) --information_schema 28 -+information_schema 42 - mysql 22 - create table t1 (i int, j int); - create trigger trg1 before insert on t1 for each row -@@ -1267,6 +1286,20 @@ - TRIGGERS TRIGGER_SCHEMA - USER_PRIVILEGES GRANTEE - VIEWS TABLE_SCHEMA -+INNODB_BUFFER_POOL_PAGES_INDEX schema_name -+INNODB_RSEG rseg_id -+INNODB_LOCKS lock_id -+INNODB_BUFFER_POOL_PAGES page_type -+XTRADB_ENHANCEMENTS name -+INNODB_TRX trx_id -+INNODB_INDEX_STATS table_name -+INNODB_LOCK_WAITS requesting_trx_id -+INNODB_CMP_RESET page_size -+INNODB_CMP page_size -+INNODB_CMPMEM_RESET page_size -+INNODB_BUFFER_POOL_PAGES_BLOB space_id -+INNODB_CMPMEM page_size -+INNODB_TABLE_STATS table_name - SELECT t.table_name, c1.column_name - FROM information_schema.tables t - INNER JOIN -@@ -1310,14 +1343,28 @@ - TRIGGERS TRIGGER_SCHEMA - USER_PRIVILEGES GRANTEE - VIEWS TABLE_SCHEMA -+INNODB_BUFFER_POOL_PAGES_INDEX schema_name -+INNODB_RSEG rseg_id -+INNODB_LOCKS lock_id -+INNODB_BUFFER_POOL_PAGES page_type -+XTRADB_ENHANCEMENTS name -+INNODB_TRX trx_id -+INNODB_INDEX_STATS table_name -+INNODB_LOCK_WAITS requesting_trx_id -+INNODB_CMP_RESET page_size -+INNODB_CMP page_size -+INNODB_CMPMEM_RESET page_size -+INNODB_BUFFER_POOL_PAGES_BLOB space_id -+INNODB_CMPMEM page_size -+INNODB_TABLE_STATS table_name - SELECT MAX(table_name) FROM information_schema.tables WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test'); - MAX(table_name) --VIEWS -+XTRADB_ENHANCEMENTS - SELECT table_name from information_schema.tables - WHERE table_name=(SELECT MAX(table_name) - FROM information_schema.tables WHERE table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test')); - table_name --VIEWS -+XTRADB_ENHANCEMENTS - DROP TABLE IF EXISTS bug23037; - DROP FUNCTION IF EXISTS get_value; - SELECT COLUMN_NAME, MD5(COLUMN_DEFAULT), LENGTH(COLUMN_DEFAULT) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME='bug23037'; -@@ -1386,6 +1433,19 @@ - FILES information_schema.FILES 1 - GLOBAL_STATUS information_schema.GLOBAL_STATUS 1 - GLOBAL_VARIABLES information_schema.GLOBAL_VARIABLES 1 -+INNODB_BUFFER_POOL_PAGES information_schema.INNODB_BUFFER_POOL_PAGES 1 -+INNODB_BUFFER_POOL_PAGES_BLOB information_schema.INNODB_BUFFER_POOL_PAGES_BLOB 1 -+INNODB_BUFFER_POOL_PAGES_INDEX information_schema.INNODB_BUFFER_POOL_PAGES_INDEX 1 -+INNODB_CMP information_schema.INNODB_CMP 1 -+INNODB_CMPMEM information_schema.INNODB_CMPMEM 1 -+INNODB_CMPMEM_RESET information_schema.INNODB_CMPMEM_RESET 1 -+INNODB_CMP_RESET information_schema.INNODB_CMP_RESET 1 -+INNODB_INDEX_STATS information_schema.INNODB_INDEX_STATS 1 -+INNODB_LOCKS information_schema.INNODB_LOCKS 1 -+INNODB_LOCK_WAITS information_schema.INNODB_LOCK_WAITS 1 -+INNODB_RSEG information_schema.INNODB_RSEG 1 -+INNODB_TABLE_STATS information_schema.INNODB_TABLE_STATS 1 -+INNODB_TRX information_schema.INNODB_TRX 1 - KEY_COLUMN_USAGE information_schema.KEY_COLUMN_USAGE 1 - PARTITIONS information_schema.PARTITIONS 1 - PLUGINS information_schema.PLUGINS 1 -@@ -1404,6 +1464,7 @@ - TRIGGERS information_schema.TRIGGERS 1 - USER_PRIVILEGES information_schema.USER_PRIVILEGES 1 - VIEWS information_schema.VIEWS 1 -+XTRADB_ENHANCEMENTS information_schema.XTRADB_ENHANCEMENTS 1 - create table t1(f1 int); - create view v1 as select f1+1 as a from t1; - create table t2 (f1 int, f2 int); === removed file 'mysql-test/patches/information_schema_db.diff' --- a/mysql-test/patches/information_schema_db.diff 2009-09-08 16:04:58 +0000 +++ b/mysql-test/patches/information_schema_db.diff 1970-01-01 00:00:00 +0000 @@ -1,23 +0,0 @@ ---- mysql-test/r/information_schema_db.result.orig 2009-06-25 21:41:38.000000000 +0000 -+++ mysql-test/r/information_schema_db.result 2009-06-25 21:42:01.000000000 +0000 -@@ -33,6 +33,20 @@ - TRIGGERS - USER_PRIVILEGES - VIEWS -+INNODB_BUFFER_POOL_PAGES_INDEX -+INNODB_RSEG -+INNODB_LOCKS -+INNODB_BUFFER_POOL_PAGES -+XTRADB_ENHANCEMENTS -+INNODB_TRX -+INNODB_INDEX_STATS -+INNODB_LOCK_WAITS -+INNODB_CMP_RESET -+INNODB_CMP -+INNODB_CMPMEM_RESET -+INNODB_BUFFER_POOL_PAGES_BLOB -+INNODB_CMPMEM -+INNODB_TABLE_STATS - show tables from INFORMATION_SCHEMA like 'T%'; - Tables_in_information_schema (T%) - TABLES === removed file 'mysql-test/patches/mysqlshow.diff' --- a/mysql-test/patches/mysqlshow.diff 2009-09-08 16:04:58 +0000 +++ b/mysql-test/patches/mysqlshow.diff 1970-01-01 00:00:00 +0000 @@ -1,44 +0,0 @@ ---- mysql-test/r/mysqlshow.result.orig 2009-06-25 21:56:22.000000000 +0000 -+++ mysql-test/r/mysqlshow.result 2009-06-25 21:56:28.000000000 +0000 -@@ -107,6 +107,20 @@ - | TRIGGERS | - | USER_PRIVILEGES | - | VIEWS | -+| INNODB_BUFFER_POOL_PAGES_INDEX | -+| INNODB_RSEG | -+| INNODB_LOCKS | -+| INNODB_BUFFER_POOL_PAGES | -+| XTRADB_ENHANCEMENTS | -+| INNODB_TRX | -+| INNODB_INDEX_STATS | -+| INNODB_LOCK_WAITS | -+| INNODB_CMP_RESET | -+| INNODB_CMP | -+| INNODB_CMPMEM_RESET | -+| INNODB_BUFFER_POOL_PAGES_BLOB | -+| INNODB_CMPMEM | -+| INNODB_TABLE_STATS | - +---------------------------------------+ - Database: INFORMATION_SCHEMA - +---------------------------------------+ -@@ -140,6 +154,20 @@ - | TRIGGERS | - | USER_PRIVILEGES | - | VIEWS | -+| INNODB_BUFFER_POOL_PAGES_INDEX | -+| INNODB_RSEG | -+| INNODB_LOCKS | -+| INNODB_BUFFER_POOL_PAGES | -+| XTRADB_ENHANCEMENTS | -+| INNODB_TRX | -+| INNODB_INDEX_STATS | -+| INNODB_LOCK_WAITS | -+| INNODB_CMP_RESET | -+| INNODB_CMP | -+| INNODB_CMPMEM_RESET | -+| INNODB_BUFFER_POOL_PAGES_BLOB | -+| INNODB_CMPMEM | -+| INNODB_TABLE_STATS | - +---------------------------------------+ - Wildcard: inf_rmation_schema - +--------------------+

1 0

[Maria-developers] Log_of_MariaDB_Contributions
by Henrik Ingo 14 Sep '09

14 Sep '09

Hi all Maria Developers I promised to do this some weeks ago: http://askmonty.org/wiki/index.php/Log_of_MariaDB_Contributions We need to include in our routines that any time a Maria Captain pushes something to the main trunk that was not 100% developed by Monty Program employees (and hopefully this will happen often :-), a log entry to that page is inserted. This is important so that we have a documented trail on how rights have been assigned for all code going into the trunk. The Linux kernel seems to have incorporated this into their git processes by adding signed-off headers to their git changeset metadata (noting that Linux of course is GPL only). If there is a way to do something similar with bzr I'm open to suggestions. If we did it with bzr, my next question would be, how can a stupid manager like me easily get a report of the signed-off trail? Similarly, feel free to modify the above wiki page if some other format works better. (For instance, we may agree with some contributor that anything available in a dedicated lp tree is contributed to MariaDB.) Sanja: it seems that the first thing that should be added to the page is the virtual columns patch you are working on. Having come from Drizzle there is a chance it is under BSD? henrik -- email: henrik.ingo(a)avoinelama.fi tel: +358-40-5697354 www: www.avoinelama.fi/~hingo book: www.openlife.cc

1 0

[Maria-developers] Updated (by Guest): Add a mysqlbinlog option to change the used database (36)
by worklog-noreply＠askmonty.org 14 Sep '09

14 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add a mysqlbinlog option to change the used database CREATION DATE..: Fri, 07 Aug 2009, 14:57 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 36 (http://askmonty.org/worklog/?tid=36) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Mon, 14 Sep 2009, 11:51)=-=- Low Level Design modified. --- /tmp/wklog.36.old.9711 2009-09-14 11:51:43.000000000 +0300 +++ /tmp/wklog.36.new.9711 2009-09-14 11:51:43.000000000 +0300 @@ -1 +1 @@ -Pay no attention: just check for having access + -=-=(Guest - Mon, 14 Sep 2009, 11:51)=-=- Low Level Design modified. --- /tmp/wklog.36.old.9678 2009-09-14 11:51:28.000000000 +0300 +++ /tmp/wklog.36.new.9678 2009-09-14 11:51:28.000000000 +0300 @@ -1 +1 @@ - +Pay no attention: just check for having access -=-=(Knielsen - Mon, 17 Aug 2009, 12:44)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.7834 2009-08-17 12:44:17.000000000 +0300 +++ /tmp/wklog.36.new.7834 2009-08-17 12:44:17.000000000 +0300 @@ -13,7 +13,9 @@ statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). -See also MySQL BUG#42941. +See also MySQL BUG#42941. Note this bug is fixed in MySQL 5.1.37, which is not +merged into MariaDB at the time of writing, but planned to be merged before +release. What we could do ---------------- -=-=(Guest - Sun, 16 Aug 2009, 17:11)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.27162 2009-08-16 17:11:12.000000000 +0300 +++ /tmp/wklog.36.new.27162 2009-08-16 17:11:12.000000000 +0300 @@ -13,6 +13,8 @@ statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). +See also MySQL BUG#42941. + What we could do ---------------- -=-=(Psergey - Mon, 10 Aug 2009, 15:41)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.13035 2009-08-10 15:41:51.000000000 +0300 +++ /tmp/wklog.36.new.13035 2009-08-10 15:41:51.000000000 +0300 @@ -1,5 +1,7 @@ Context ------- +(See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global +overview) At the moment, the server has a replication slave option --replicate-rewrite-db="from->to" -=-=(Guest - Mon, 10 Aug 2009, 11:12)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.6580 2009-08-10 11:12:36.000000000 +0300 +++ /tmp/wklog.36.new.6580 2009-08-10 11:12:36.000000000 +0300 @@ -1,4 +1,3 @@ - Context ------- At the moment, the server has a replication slave option @@ -67,6 +66,6 @@ It will be possible to do the rewrites either on the slave ( --replicate-rewrite-db will work for all kinds of statements), or in -mysqlbinlog (adding a comment is easy and doesn't require use to parse the -statement). +mysqlbinlog (adding a comment is easy and doesn't require mysqlbinlog to +parse the statement). -=-=(Psergey - Sun, 09 Aug 2009, 23:53)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.13425 2009-08-09 23:53:54.000000000 +0300 +++ /tmp/wklog.36.new.13425 2009-08-09 23:53:54.000000000 +0300 @@ -1 +1,72 @@ +Context +------- +At the moment, the server has a replication slave option + + --replicate-rewrite-db="from->to" + +the option affects +- Table_map_log_event (all RBR events) +- Load_log_event (LOAD DATA) +- Query_log_event (SBR-based updates, with the usual assumption that the + statement refers to tables in current database, so that changing the current + database will make the statement to work on a table in a different database). + +What we could do +---------------- + +Option1: make mysqlbinlog accept --replicate-rewrite-db option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Make mysqlbinlog accept --replicate-rewrite-db options and process them to the +same extent as replication slave would process --replicate-rewrite-db option. + + +Option2: Add database-agnostic RBR events and --strip-db option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Right now RBR events require a databasename. It is not possible to have RBR +event stream that won't mention which database the events are for. When I +tried to use debugger and specify empty database name, attempt to apply the +binlog resulted in this error: + +090809 17:38:44 [ERROR] Slave SQL: Error 'Table '.tablename' doesn't exist' on +opening tables, + +We could do as follows: +- Make the server interpret empty database name in RBR event (i.e. in a + Table_map_log_event) as "use current database". Binlog slave thread + probably should not allow such events as it doesn't have a natural current + database. +- Add a mysqlbinlog --strip-db option that would + = not produce any "USE dbname" statements + = change databasename for all RBR events to be empty + +That way, mysqlbinlog output will be database-agnostic and apply to the +current database. +(this will have the usual limitations that we assume that all statements in +the binlog refer to the current database). + +Option3: Enhance database rewrite +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +If there is a need to support database change for statements that use +dbname.tablename notation and are replicated as statements (i.e. are DDL +statements and/or DML statements that are binlogged as statements), +then that could be supported as follows: + +- Make the server's parser recognize special form of comments + + /* !database-alias(oldname,newname) */ + + and save the mapping somewhere + +- Put the hooks in table open and name resolution code to use the saved + mapping. + + +Once we've done the above, it will be easy to perform a complete, +no-compromise or restrictions database name change in binary log. + +It will be possible to do the rewrites either on the slave ( +--replicate-rewrite-db will work for all kinds of statements), or in +mysqlbinlog (adding a comment is easy and doesn't require use to parse the +statement). + -=-=(Psergey - Sun, 09 Aug 2009, 12:27)=-=- Dependency created: 39 now depends on 36 -=-=(Psergey - Fri, 07 Aug 2009, 14:57)=-=- Title modified. --- /tmp/wklog.36.old.14687 2009-08-07 14:57:49.000000000 +0300 +++ /tmp/wklog.36.new.14687 2009-08-07 14:57:49.000000000 +0300 @@ -1 +1 @@ -Add a mysqlbinlog option to change the database +Add a mysqlbinlog option to change the used database DESCRIPTION: Sometimes there is a need to take a binary log and apply it to a database with a different name than the original name of the database on binlog producer. If one is using statement-based replication, he can achieve this by grepping out "USE dbname" statements out of the output of mysqlbinlog(*). With row-based replication this is no longer possible, as database name is encoded within the the BINLOG '....' statement. This task is about adding an option to mysqlbinlog that would allow to change the names of used databases in both RBR and SBR events. (*) this implies that all statements refer to tables in the current database, doesn't catch updates made inside stored functions and so forth, but still works for a practially-important subset of cases. HIGH-LEVEL SPECIFICATION: Context ------- (See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global overview) At the moment, the server has a replication slave option --replicate-rewrite-db="from->to" the option affects - Table_map_log_event (all RBR events) - Load_log_event (LOAD DATA) - Query_log_event (SBR-based updates, with the usual assumption that the statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). See also MySQL BUG#42941. Note this bug is fixed in MySQL 5.1.37, which is not merged into MariaDB at the time of writing, but planned to be merged before release. What we could do ---------------- Option1: make mysqlbinlog accept --replicate-rewrite-db option ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Make mysqlbinlog accept --replicate-rewrite-db options and process them to the same extent as replication slave would process --replicate-rewrite-db option. Option2: Add database-agnostic RBR events and --strip-db option ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Right now RBR events require a databasename. It is not possible to have RBR event stream that won't mention which database the events are for. When I tried to use debugger and specify empty database name, attempt to apply the binlog resulted in this error: 090809 17:38:44 [ERROR] Slave SQL: Error 'Table '.tablename' doesn't exist' on opening tables, We could do as follows: - Make the server interpret empty database name in RBR event (i.e. in a Table_map_log_event) as "use current database". Binlog slave thread probably should not allow such events as it doesn't have a natural current database. - Add a mysqlbinlog --strip-db option that would = not produce any "USE dbname" statements = change databasename for all RBR events to be empty That way, mysqlbinlog output will be database-agnostic and apply to the current database. (this will have the usual limitations that we assume that all statements in the binlog refer to the current database). Option3: Enhance database rewrite ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If there is a need to support database change for statements that use dbname.tablename notation and are replicated as statements (i.e. are DDL statements and/or DML statements that are binlogged as statements), then that could be supported as follows: - Make the server's parser recognize special form of comments /* !database-alias(oldname,newname) */ and save the mapping somewhere - Put the hooks in table open and name resolution code to use the saved mapping. Once we've done the above, it will be easy to perform a complete, no-compromise or restrictions database name change in binary log. It will be possible to do the rewrites either on the slave ( --replicate-rewrite-db will work for all kinds of statements), or in mysqlbinlog (adding a comment is easy and doesn't require mysqlbinlog to parse the statement). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Add a mysqlbinlog option to change the used database (36)
by worklog-noreply＠askmonty.org 14 Sep '09

14 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add a mysqlbinlog option to change the used database CREATION DATE..: Fri, 07 Aug 2009, 14:57 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 36 (http://askmonty.org/worklog/?tid=36) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Mon, 14 Sep 2009, 11:51)=-=- Low Level Design modified. --- /tmp/wklog.36.old.9711 2009-09-14 11:51:43.000000000 +0300 +++ /tmp/wklog.36.new.9711 2009-09-14 11:51:43.000000000 +0300 @@ -1 +1 @@ -Pay no attention: just check for having access + -=-=(Guest - Mon, 14 Sep 2009, 11:51)=-=- Low Level Design modified. --- /tmp/wklog.36.old.9678 2009-09-14 11:51:28.000000000 +0300 +++ /tmp/wklog.36.new.9678 2009-09-14 11:51:28.000000000 +0300 @@ -1 +1 @@ - +Pay no attention: just check for having access -=-=(Knielsen - Mon, 17 Aug 2009, 12:44)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.7834 2009-08-17 12:44:17.000000000 +0300 +++ /tmp/wklog.36.new.7834 2009-08-17 12:44:17.000000000 +0300 @@ -13,7 +13,9 @@ statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). -See also MySQL BUG#42941. +See also MySQL BUG#42941. Note this bug is fixed in MySQL 5.1.37, which is not +merged into MariaDB at the time of writing, but planned to be merged before +release. What we could do ---------------- -=-=(Guest - Sun, 16 Aug 2009, 17:11)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.27162 2009-08-16 17:11:12.000000000 +0300 +++ /tmp/wklog.36.new.27162 2009-08-16 17:11:12.000000000 +0300 @@ -13,6 +13,8 @@ statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). +See also MySQL BUG#42941. + What we could do ---------------- -=-=(Psergey - Mon, 10 Aug 2009, 15:41)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.13035 2009-08-10 15:41:51.000000000 +0300 +++ /tmp/wklog.36.new.13035 2009-08-10 15:41:51.000000000 +0300 @@ -1,5 +1,7 @@ Context ------- +(See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global +overview) At the moment, the server has a replication slave option --replicate-rewrite-db="from->to" -=-=(Guest - Mon, 10 Aug 2009, 11:12)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.6580 2009-08-10 11:12:36.000000000 +0300 +++ /tmp/wklog.36.new.6580 2009-08-10 11:12:36.000000000 +0300 @@ -1,4 +1,3 @@ - Context ------- At the moment, the server has a replication slave option @@ -67,6 +66,6 @@ It will be possible to do the rewrites either on the slave ( --replicate-rewrite-db will work for all kinds of statements), or in -mysqlbinlog (adding a comment is easy and doesn't require use to parse the -statement). +mysqlbinlog (adding a comment is easy and doesn't require mysqlbinlog to +parse the statement). -=-=(Psergey - Sun, 09 Aug 2009, 23:53)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.13425 2009-08-09 23:53:54.000000000 +0300 +++ /tmp/wklog.36.new.13425 2009-08-09 23:53:54.000000000 +0300 @@ -1 +1,72 @@ +Context +------- +At the moment, the server has a replication slave option + + --replicate-rewrite-db="from->to" + +the option affects +- Table_map_log_event (all RBR events) +- Load_log_event (LOAD DATA) +- Query_log_event (SBR-based updates, with the usual assumption that the + statement refers to tables in current database, so that changing the current + database will make the statement to work on a table in a different database). + +What we could do +---------------- + +Option1: make mysqlbinlog accept --replicate-rewrite-db option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Make mysqlbinlog accept --replicate-rewrite-db options and process them to the +same extent as replication slave would process --replicate-rewrite-db option. + + +Option2: Add database-agnostic RBR events and --strip-db option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Right now RBR events require a databasename. It is not possible to have RBR +event stream that won't mention which database the events are for. When I +tried to use debugger and specify empty database name, attempt to apply the +binlog resulted in this error: + +090809 17:38:44 [ERROR] Slave SQL: Error 'Table '.tablename' doesn't exist' on +opening tables, + +We could do as follows: +- Make the server interpret empty database name in RBR event (i.e. in a + Table_map_log_event) as "use current database". Binlog slave thread + probably should not allow such events as it doesn't have a natural current + database. +- Add a mysqlbinlog --strip-db option that would + = not produce any "USE dbname" statements + = change databasename for all RBR events to be empty + +That way, mysqlbinlog output will be database-agnostic and apply to the +current database. +(this will have the usual limitations that we assume that all statements in +the binlog refer to the current database). + +Option3: Enhance database rewrite +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +If there is a need to support database change for statements that use +dbname.tablename notation and are replicated as statements (i.e. are DDL +statements and/or DML statements that are binlogged as statements), +then that could be supported as follows: + +- Make the server's parser recognize special form of comments + + /* !database-alias(oldname,newname) */ + + and save the mapping somewhere + +- Put the hooks in table open and name resolution code to use the saved + mapping. + + +Once we've done the above, it will be easy to perform a complete, +no-compromise or restrictions database name change in binary log. + +It will be possible to do the rewrites either on the slave ( +--replicate-rewrite-db will work for all kinds of statements), or in +mysqlbinlog (adding a comment is easy and doesn't require use to parse the +statement). + -=-=(Psergey - Sun, 09 Aug 2009, 12:27)=-=- Dependency created: 39 now depends on 36 -=-=(Psergey - Fri, 07 Aug 2009, 14:57)=-=- Title modified. --- /tmp/wklog.36.old.14687 2009-08-07 14:57:49.000000000 +0300 +++ /tmp/wklog.36.new.14687 2009-08-07 14:57:49.000000000 +0300 @@ -1 +1 @@ -Add a mysqlbinlog option to change the database +Add a mysqlbinlog option to change the used database DESCRIPTION: Sometimes there is a need to take a binary log and apply it to a database with a different name than the original name of the database on binlog producer. If one is using statement-based replication, he can achieve this by grepping out "USE dbname" statements out of the output of mysqlbinlog(*). With row-based replication this is no longer possible, as database name is encoded within the the BINLOG '....' statement. This task is about adding an option to mysqlbinlog that would allow to change the names of used databases in both RBR and SBR events. (*) this implies that all statements refer to tables in the current database, doesn't catch updates made inside stored functions and so forth, but still works for a practially-important subset of cases. HIGH-LEVEL SPECIFICATION: Context ------- (See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global overview) At the moment, the server has a replication slave option --replicate-rewrite-db="from->to" the option affects - Table_map_log_event (all RBR events) - Load_log_event (LOAD DATA) - Query_log_event (SBR-based updates, with the usual assumption that the statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). See also MySQL BUG#42941. Note this bug is fixed in MySQL 5.1.37, which is not merged into MariaDB at the time of writing, but planned to be merged before release. What we could do ---------------- Option1: make mysqlbinlog accept --replicate-rewrite-db option ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Make mysqlbinlog accept --replicate-rewrite-db options and process them to the same extent as replication slave would process --replicate-rewrite-db option. Option2: Add database-agnostic RBR events and --strip-db option ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Right now RBR events require a databasename. It is not possible to have RBR event stream that won't mention which database the events are for. When I tried to use debugger and specify empty database name, attempt to apply the binlog resulted in this error: 090809 17:38:44 [ERROR] Slave SQL: Error 'Table '.tablename' doesn't exist' on opening tables, We could do as follows: - Make the server interpret empty database name in RBR event (i.e. in a Table_map_log_event) as "use current database". Binlog slave thread probably should not allow such events as it doesn't have a natural current database. - Add a mysqlbinlog --strip-db option that would = not produce any "USE dbname" statements = change databasename for all RBR events to be empty That way, mysqlbinlog output will be database-agnostic and apply to the current database. (this will have the usual limitations that we assume that all statements in the binlog refer to the current database). Option3: Enhance database rewrite ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If there is a need to support database change for statements that use dbname.tablename notation and are replicated as statements (i.e. are DDL statements and/or DML statements that are binlogged as statements), then that could be supported as follows: - Make the server's parser recognize special form of comments /* !database-alias(oldname,newname) */ and save the mapping somewhere - Put the hooks in table open and name resolution code to use the saved mapping. Once we've done the above, it will be easy to perform a complete, no-compromise or restrictions database name change in binary log. It will be possible to do the rewrites either on the slave ( --replicate-rewrite-db will work for all kinds of statements), or in mysqlbinlog (adding a comment is easy and doesn't require mysqlbinlog to parse the statement). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Add a mysqlbinlog option to change the used database (36)
by worklog-noreply＠askmonty.org 14 Sep '09

14 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add a mysqlbinlog option to change the used database CREATION DATE..: Fri, 07 Aug 2009, 14:57 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 36 (http://askmonty.org/worklog/?tid=36) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Mon, 14 Sep 2009, 11:51)=-=- Low Level Design modified. --- /tmp/wklog.36.old.9678 2009-09-14 11:51:28.000000000 +0300 +++ /tmp/wklog.36.new.9678 2009-09-14 11:51:28.000000000 +0300 @@ -1 +1 @@ - +Pay no attention: just check for having access -=-=(Knielsen - Mon, 17 Aug 2009, 12:44)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.7834 2009-08-17 12:44:17.000000000 +0300 +++ /tmp/wklog.36.new.7834 2009-08-17 12:44:17.000000000 +0300 @@ -13,7 +13,9 @@ statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). -See also MySQL BUG#42941. +See also MySQL BUG#42941. Note this bug is fixed in MySQL 5.1.37, which is not +merged into MariaDB at the time of writing, but planned to be merged before +release. What we could do ---------------- -=-=(Guest - Sun, 16 Aug 2009, 17:11)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.27162 2009-08-16 17:11:12.000000000 +0300 +++ /tmp/wklog.36.new.27162 2009-08-16 17:11:12.000000000 +0300 @@ -13,6 +13,8 @@ statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). +See also MySQL BUG#42941. + What we could do ---------------- -=-=(Psergey - Mon, 10 Aug 2009, 15:41)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.13035 2009-08-10 15:41:51.000000000 +0300 +++ /tmp/wklog.36.new.13035 2009-08-10 15:41:51.000000000 +0300 @@ -1,5 +1,7 @@ Context ------- +(See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global +overview) At the moment, the server has a replication slave option --replicate-rewrite-db="from->to" -=-=(Guest - Mon, 10 Aug 2009, 11:12)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.6580 2009-08-10 11:12:36.000000000 +0300 +++ /tmp/wklog.36.new.6580 2009-08-10 11:12:36.000000000 +0300 @@ -1,4 +1,3 @@ - Context ------- At the moment, the server has a replication slave option @@ -67,6 +66,6 @@ It will be possible to do the rewrites either on the slave ( --replicate-rewrite-db will work for all kinds of statements), or in -mysqlbinlog (adding a comment is easy and doesn't require use to parse the -statement). +mysqlbinlog (adding a comment is easy and doesn't require mysqlbinlog to +parse the statement). -=-=(Psergey - Sun, 09 Aug 2009, 23:53)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.13425 2009-08-09 23:53:54.000000000 +0300 +++ /tmp/wklog.36.new.13425 2009-08-09 23:53:54.000000000 +0300 @@ -1 +1,72 @@ +Context +------- +At the moment, the server has a replication slave option + + --replicate-rewrite-db="from->to" + +the option affects +- Table_map_log_event (all RBR events) +- Load_log_event (LOAD DATA) +- Query_log_event (SBR-based updates, with the usual assumption that the + statement refers to tables in current database, so that changing the current + database will make the statement to work on a table in a different database). + +What we could do +---------------- + +Option1: make mysqlbinlog accept --replicate-rewrite-db option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Make mysqlbinlog accept --replicate-rewrite-db options and process them to the +same extent as replication slave would process --replicate-rewrite-db option. + + +Option2: Add database-agnostic RBR events and --strip-db option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Right now RBR events require a databasename. It is not possible to have RBR +event stream that won't mention which database the events are for. When I +tried to use debugger and specify empty database name, attempt to apply the +binlog resulted in this error: + +090809 17:38:44 [ERROR] Slave SQL: Error 'Table '.tablename' doesn't exist' on +opening tables, + +We could do as follows: +- Make the server interpret empty database name in RBR event (i.e. in a + Table_map_log_event) as "use current database". Binlog slave thread + probably should not allow such events as it doesn't have a natural current + database. +- Add a mysqlbinlog --strip-db option that would + = not produce any "USE dbname" statements + = change databasename for all RBR events to be empty + +That way, mysqlbinlog output will be database-agnostic and apply to the +current database. +(this will have the usual limitations that we assume that all statements in +the binlog refer to the current database). + +Option3: Enhance database rewrite +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +If there is a need to support database change for statements that use +dbname.tablename notation and are replicated as statements (i.e. are DDL +statements and/or DML statements that are binlogged as statements), +then that could be supported as follows: + +- Make the server's parser recognize special form of comments + + /* !database-alias(oldname,newname) */ + + and save the mapping somewhere + +- Put the hooks in table open and name resolution code to use the saved + mapping. + + +Once we've done the above, it will be easy to perform a complete, +no-compromise or restrictions database name change in binary log. + +It will be possible to do the rewrites either on the slave ( +--replicate-rewrite-db will work for all kinds of statements), or in +mysqlbinlog (adding a comment is easy and doesn't require use to parse the +statement). + -=-=(Psergey - Sun, 09 Aug 2009, 12:27)=-=- Dependency created: 39 now depends on 36 -=-=(Psergey - Fri, 07 Aug 2009, 14:57)=-=- Title modified. --- /tmp/wklog.36.old.14687 2009-08-07 14:57:49.000000000 +0300 +++ /tmp/wklog.36.new.14687 2009-08-07 14:57:49.000000000 +0300 @@ -1 +1 @@ -Add a mysqlbinlog option to change the database +Add a mysqlbinlog option to change the used database DESCRIPTION: Sometimes there is a need to take a binary log and apply it to a database with a different name than the original name of the database on binlog producer. If one is using statement-based replication, he can achieve this by grepping out "USE dbname" statements out of the output of mysqlbinlog(*). With row-based replication this is no longer possible, as database name is encoded within the the BINLOG '....' statement. This task is about adding an option to mysqlbinlog that would allow to change the names of used databases in both RBR and SBR events. (*) this implies that all statements refer to tables in the current database, doesn't catch updates made inside stored functions and so forth, but still works for a practially-important subset of cases. HIGH-LEVEL SPECIFICATION: Context ------- (See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global overview) At the moment, the server has a replication slave option --replicate-rewrite-db="from->to" the option affects - Table_map_log_event (all RBR events) - Load_log_event (LOAD DATA) - Query_log_event (SBR-based updates, with the usual assumption that the statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). See also MySQL BUG#42941. Note this bug is fixed in MySQL 5.1.37, which is not merged into MariaDB at the time of writing, but planned to be merged before release. What we could do ---------------- Option1: make mysqlbinlog accept --replicate-rewrite-db option ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Make mysqlbinlog accept --replicate-rewrite-db options and process them to the same extent as replication slave would process --replicate-rewrite-db option. Option2: Add database-agnostic RBR events and --strip-db option ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Right now RBR events require a databasename. It is not possible to have RBR event stream that won't mention which database the events are for. When I tried to use debugger and specify empty database name, attempt to apply the binlog resulted in this error: 090809 17:38:44 [ERROR] Slave SQL: Error 'Table '.tablename' doesn't exist' on opening tables, We could do as follows: - Make the server interpret empty database name in RBR event (i.e. in a Table_map_log_event) as "use current database". Binlog slave thread probably should not allow such events as it doesn't have a natural current database. - Add a mysqlbinlog --strip-db option that would = not produce any "USE dbname" statements = change databasename for all RBR events to be empty That way, mysqlbinlog output will be database-agnostic and apply to the current database. (this will have the usual limitations that we assume that all statements in the binlog refer to the current database). Option3: Enhance database rewrite ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If there is a need to support database change for statements that use dbname.tablename notation and are replicated as statements (i.e. are DDL statements and/or DML statements that are binlogged as statements), then that could be supported as follows: - Make the server's parser recognize special form of comments /* !database-alias(oldname,newname) */ and save the mapping somewhere - Put the hooks in table open and name resolution code to use the saved mapping. Once we've done the above, it will be easy to perform a complete, no-compromise or restrictions database name change in binary log. It will be possible to do the rewrites either on the slave ( --replicate-rewrite-db will work for all kinds of statements), or in mysqlbinlog (adding a comment is easy and doesn't require mysqlbinlog to parse the statement). LOW-LEVEL DESIGN: Pay no attention: just check for having access ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Add a mysqlbinlog option to change the used database (36)
by worklog-noreply＠askmonty.org 14 Sep '09

14 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add a mysqlbinlog option to change the used database CREATION DATE..: Fri, 07 Aug 2009, 14:57 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 36 (http://askmonty.org/worklog/?tid=36) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Mon, 14 Sep 2009, 11:51)=-=- Low Level Design modified. --- /tmp/wklog.36.old.9678 2009-09-14 11:51:28.000000000 +0300 +++ /tmp/wklog.36.new.9678 2009-09-14 11:51:28.000000000 +0300 @@ -1 +1 @@ - +Pay no attention: just check for having access -=-=(Knielsen - Mon, 17 Aug 2009, 12:44)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.7834 2009-08-17 12:44:17.000000000 +0300 +++ /tmp/wklog.36.new.7834 2009-08-17 12:44:17.000000000 +0300 @@ -13,7 +13,9 @@ statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). -See also MySQL BUG#42941. +See also MySQL BUG#42941. Note this bug is fixed in MySQL 5.1.37, which is not +merged into MariaDB at the time of writing, but planned to be merged before +release. What we could do ---------------- -=-=(Guest - Sun, 16 Aug 2009, 17:11)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.27162 2009-08-16 17:11:12.000000000 +0300 +++ /tmp/wklog.36.new.27162 2009-08-16 17:11:12.000000000 +0300 @@ -13,6 +13,8 @@ statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). +See also MySQL BUG#42941. + What we could do ---------------- -=-=(Psergey - Mon, 10 Aug 2009, 15:41)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.13035 2009-08-10 15:41:51.000000000 +0300 +++ /tmp/wklog.36.new.13035 2009-08-10 15:41:51.000000000 +0300 @@ -1,5 +1,7 @@ Context ------- +(See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global +overview) At the moment, the server has a replication slave option --replicate-rewrite-db="from->to" -=-=(Guest - Mon, 10 Aug 2009, 11:12)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.6580 2009-08-10 11:12:36.000000000 +0300 +++ /tmp/wklog.36.new.6580 2009-08-10 11:12:36.000000000 +0300 @@ -1,4 +1,3 @@ - Context ------- At the moment, the server has a replication slave option @@ -67,6 +66,6 @@ It will be possible to do the rewrites either on the slave ( --replicate-rewrite-db will work for all kinds of statements), or in -mysqlbinlog (adding a comment is easy and doesn't require use to parse the -statement). +mysqlbinlog (adding a comment is easy and doesn't require mysqlbinlog to +parse the statement). -=-=(Psergey - Sun, 09 Aug 2009, 23:53)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.13425 2009-08-09 23:53:54.000000000 +0300 +++ /tmp/wklog.36.new.13425 2009-08-09 23:53:54.000000000 +0300 @@ -1 +1,72 @@ +Context +------- +At the moment, the server has a replication slave option + + --replicate-rewrite-db="from->to" + +the option affects +- Table_map_log_event (all RBR events) +- Load_log_event (LOAD DATA) +- Query_log_event (SBR-based updates, with the usual assumption that the + statement refers to tables in current database, so that changing the current + database will make the statement to work on a table in a different database). + +What we could do +---------------- + +Option1: make mysqlbinlog accept --replicate-rewrite-db option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Make mysqlbinlog accept --replicate-rewrite-db options and process them to the +same extent as replication slave would process --replicate-rewrite-db option. + + +Option2: Add database-agnostic RBR events and --strip-db option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Right now RBR events require a databasename. It is not possible to have RBR +event stream that won't mention which database the events are for. When I +tried to use debugger and specify empty database name, attempt to apply the +binlog resulted in this error: + +090809 17:38:44 [ERROR] Slave SQL: Error 'Table '.tablename' doesn't exist' on +opening tables, + +We could do as follows: +- Make the server interpret empty database name in RBR event (i.e. in a + Table_map_log_event) as "use current database". Binlog slave thread + probably should not allow such events as it doesn't have a natural current + database. +- Add a mysqlbinlog --strip-db option that would + = not produce any "USE dbname" statements + = change databasename for all RBR events to be empty + +That way, mysqlbinlog output will be database-agnostic and apply to the +current database. +(this will have the usual limitations that we assume that all statements in +the binlog refer to the current database). + +Option3: Enhance database rewrite +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +If there is a need to support database change for statements that use +dbname.tablename notation and are replicated as statements (i.e. are DDL +statements and/or DML statements that are binlogged as statements), +then that could be supported as follows: + +- Make the server's parser recognize special form of comments + + /* !database-alias(oldname,newname) */ + + and save the mapping somewhere + +- Put the hooks in table open and name resolution code to use the saved + mapping. + + +Once we've done the above, it will be easy to perform a complete, +no-compromise or restrictions database name change in binary log. + +It will be possible to do the rewrites either on the slave ( +--replicate-rewrite-db will work for all kinds of statements), or in +mysqlbinlog (adding a comment is easy and doesn't require use to parse the +statement). + -=-=(Psergey - Sun, 09 Aug 2009, 12:27)=-=- Dependency created: 39 now depends on 36 -=-=(Psergey - Fri, 07 Aug 2009, 14:57)=-=- Title modified. --- /tmp/wklog.36.old.14687 2009-08-07 14:57:49.000000000 +0300 +++ /tmp/wklog.36.new.14687 2009-08-07 14:57:49.000000000 +0300 @@ -1 +1 @@ -Add a mysqlbinlog option to change the database +Add a mysqlbinlog option to change the used database DESCRIPTION: Sometimes there is a need to take a binary log and apply it to a database with a different name than the original name of the database on binlog producer. If one is using statement-based replication, he can achieve this by grepping out "USE dbname" statements out of the output of mysqlbinlog(*). With row-based replication this is no longer possible, as database name is encoded within the the BINLOG '....' statement. This task is about adding an option to mysqlbinlog that would allow to change the names of used databases in both RBR and SBR events. (*) this implies that all statements refer to tables in the current database, doesn't catch updates made inside stored functions and so forth, but still works for a practially-important subset of cases. HIGH-LEVEL SPECIFICATION: Context ------- (See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global overview) At the moment, the server has a replication slave option --replicate-rewrite-db="from->to" the option affects - Table_map_log_event (all RBR events) - Load_log_event (LOAD DATA) - Query_log_event (SBR-based updates, with the usual assumption that the statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). See also MySQL BUG#42941. Note this bug is fixed in MySQL 5.1.37, which is not merged into MariaDB at the time of writing, but planned to be merged before release. What we could do ---------------- Option1: make mysqlbinlog accept --replicate-rewrite-db option ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Make mysqlbinlog accept --replicate-rewrite-db options and process them to the same extent as replication slave would process --replicate-rewrite-db option. Option2: Add database-agnostic RBR events and --strip-db option ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Right now RBR events require a databasename. It is not possible to have RBR event stream that won't mention which database the events are for. When I tried to use debugger and specify empty database name, attempt to apply the binlog resulted in this error: 090809 17:38:44 [ERROR] Slave SQL: Error 'Table '.tablename' doesn't exist' on opening tables, We could do as follows: - Make the server interpret empty database name in RBR event (i.e. in a Table_map_log_event) as "use current database". Binlog slave thread probably should not allow such events as it doesn't have a natural current database. - Add a mysqlbinlog --strip-db option that would = not produce any "USE dbname" statements = change databasename for all RBR events to be empty That way, mysqlbinlog output will be database-agnostic and apply to the current database. (this will have the usual limitations that we assume that all statements in the binlog refer to the current database). Option3: Enhance database rewrite ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If there is a need to support database change for statements that use dbname.tablename notation and are replicated as statements (i.e. are DDL statements and/or DML statements that are binlogged as statements), then that could be supported as follows: - Make the server's parser recognize special form of comments /* !database-alias(oldname,newname) */ and save the mapping somewhere - Put the hooks in table open and name resolution code to use the saved mapping. Once we've done the above, it will be easy to perform a complete, no-compromise or restrictions database name change in binary log. It will be possible to do the rewrites either on the slave ( --replicate-rewrite-db will work for all kinds of statements), or in mysqlbinlog (adding a comment is easy and doesn't require mysqlbinlog to parse the statement). LOW-LEVEL DESIGN: Pay no attention: just check for having access ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Inventory of 3rd party GPL code in MariaDB (other than Sun)
by Henrik Ingo 13 Sep '09

13 Sep '09

Hi Related to the work I'm doing with the EU, it seems wise to conduct a review of what 3rd party code is used in MySQL/MariaDB and under what license. In other words I need to know: Things in MySQL not owned by Sun. I so far come to think of 2 modules: Readline library used in MySQL client. GPLv3 (presumably MySQL is using an earlier version under GPLv2) YaSSL for secure client server and replication connections. GPLv2 Anything else? henrik -- email: henrik.ingo(a)avoinelama.fi tel: +358-40-5697354 www: www.avoinelama.fi/~hingo book: www.openlife.cc

1 1

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2729)
by knielsen＠knielsen-hq.org 12 Sep '09

12 Sep '09

#At lp:maria 2729 knielsen(a)knielsen-hq.org 2009-09-12 Don't test innodb-use-sys-malloc under Valgrind. InnoDB does not bother to free explicitly one-time allocations at exit, so we get lots of "still reachable" warnings in Valgrind with innodb-use-sys-malloc enabled. modified: mysql-test/suite/innodb/t/innodb-use-sys-malloc.test mysql-test/t/innodb-use-sys-malloc.test === modified file 'mysql-test/suite/innodb/t/innodb-use-sys-malloc.test' --- a/mysql-test/suite/innodb/t/innodb-use-sys-malloc.test 2009-06-24 17:18:58 +0000 +++ b/mysql-test/suite/innodb/t/innodb-use-sys-malloc.test 2009-09-12 07:38:39 +0000 @@ -1,5 +1,8 @@ --source include/have_innodb.inc -- source suite/innodb/include/have_innodb_plugin.inc +# XtraDB has lots of "still reachable" memory leak warnings at shutdown when +# --innodb-use-sys-malloc +--source include/not_valgrind.inc #display current value of innodb_use_sys_malloc SELECT @@GLOBAL.innodb_use_sys_malloc; === modified file 'mysql-test/t/innodb-use-sys-malloc.test' --- a/mysql-test/t/innodb-use-sys-malloc.test 2009-06-18 12:39:21 +0000 +++ b/mysql-test/t/innodb-use-sys-malloc.test 2009-09-12 07:38:39 +0000 @@ -1,5 +1,5 @@ --source include/have_innodb.inc -# XtraDB has lots of memory leak warnings at shutdown when +# XtraDB has lots of "still reachable" memory leak warnings at shutdown when # --innodb-use-sys-malloc --source include/not_valgrind.inc

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2728)
by knielsen＠knielsen-hq.org 11 Sep '09

11 Sep '09

#At lp:maria 2728 knielsen(a)knielsen-hq.org 2009-09-11 After-merge fix for MySQL 5.1.38 merge into MariaDB. Due to a bugfix for enum options in MariaDB, my_getopt parses enums into an ulong. However, some new code from MySQL was written to assume enums take an uint. Fix by using the correct type. (The new MySQL code in addition had an implicit assumption that my_bool and uint were compatible; remove this assumption). modified: mysys/my_getopt.c sql/sql_plugin.cc === modified file 'mysys/my_getopt.c' --- a/mysys/my_getopt.c 2009-09-07 20:50:10 +0000 +++ b/mysys/my_getopt.c 2009-09-11 13:20:03 +0000 @@ -414,11 +414,17 @@ invalid value '%s'", (optp->var_type & GET_TYPE_MASK) == GET_ENUM)) { if (optend == disabled_my_option) - *((my_bool*) value)= (my_bool) 0; + if ((optp->var_type & GET_TYPE_MASK) == GET_BOOL) + *((my_bool*) value)= (my_bool) 0; + else + *((ulong*) value)= (ulong) 0; else { if (!optend) /* No argument -> enable option */ - *((my_bool*) value)= (my_bool) 1; + if ((optp->var_type & GET_TYPE_MASK) == GET_BOOL) + *((my_bool*) value)= (my_bool) 1; + else + *((ulong*) value)= (ulong) 1; else argument= optend; } === modified file 'sql/sql_plugin.cc' --- a/sql/sql_plugin.cc 2009-09-07 20:50:10 +0000 +++ b/sql/sql_plugin.cc 2009-09-11 13:20:03 +0000 @@ -2980,12 +2980,12 @@ static int construct_options(MEM_ROOT *m Allocate temporary space for the value of the tristate. This option will have a limited lifetime and is not used beyond server initialization. - GET_ENUM value is an integer. + GET_ENUM value is an ulong. */ options[0].value= options[1].value= (uchar **)alloc_root(mem_root, - sizeof(int)); - *((uint*) options[0].value)= *((uint*) options[1].value)= - (uint) options[0].def_value; + sizeof(ulong)); + *((ulong*) options[0].value)= *((ulong*) options[1].value)= + (ulong) options[0].def_value; options+= 2; @@ -3269,7 +3269,7 @@ static int test_plugin_options(MEM_ROOT Set plugin loading policy from option value. First element in the option list is always the <plugin name> option value. */ - plugin_load_policy= (enum_plugin_load_policy)*(uint*)opts[0].value; + plugin_load_policy= (enum_plugin_load_policy)*(ulong*)opts[0].value; } disable_plugin= (plugin_load_policy == PLUGIN_OFF);

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2727)
by knielsen＠knielsen-hq.org 11 Sep '09

11 Sep '09

#At lp:maria 2727 knielsen(a)knielsen-hq.org 2009-09-11 After-merge fixes for merge of MySQL-5.1.38. modified: mysql-test/r/innodb.result mysql-test/t/bug46080-master.opt mysql-test/t/innodb.test sql/sql_select.cc support-files/build-tags per-file messages: mysql-test/r/innodb.result Fix wrong merge of result file conflicts. mysql-test/t/bug46080-master.opt Fix memory limit to take into account extra storage engines in MariaDB. mysql-test/t/innodb.test Add comment clarifying tricky --replace_result. sql/sql_select.cc Restore fix of Bug#34374, accudentally lost during MySQL 5.1.38 merge. support-files/build-tags Fix missing -R option for bzr, accidentally lost in MySQL-5.1.38 merge. === modified file 'mysql-test/r/innodb.result' --- a/mysql-test/r/innodb.result 2009-09-07 20:50:10 +0000 +++ b/mysql-test/r/innodb.result 2009-09-11 11:00:53 +0000 @@ -1738,7 +1738,7 @@ count(*) drop table t1; SELECT variable_value FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_buffer_pool_pages_total'; variable_value -512 +511 SELECT variable_value FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_page_size'; variable_value 16384 @@ -1784,7 +1784,7 @@ innodb_sync_spin_loops 20 SET @old_innodb_thread_concurrency= @@global.innodb_thread_concurrency; show variables like "innodb_thread_concurrency"; Variable_name Value -innodb_thread_concurrency 8 +innodb_thread_concurrency 0 set global innodb_thread_concurrency=1001; Warnings: Warning 1292 Truncated incorrect thread_concurrency value: '1001' === modified file 'mysql-test/t/bug46080-master.opt' --- a/mysql-test/t/bug46080-master.opt 2009-07-10 12:00:34 +0000 +++ b/mysql-test/t/bug46080-master.opt 2009-09-11 11:00:53 +0000 @@ -1 +1 @@ ---skip-grant-tables --skip-name-resolve --safemalloc-mem-limit=4000000 +--skip-grant-tables --skip-name-resolve --safemalloc-mem-limit=20000000 === modified file 'mysql-test/t/innodb.test' --- a/mysql-test/t/innodb.test 2009-09-07 20:50:10 +0000 +++ b/mysql-test/t/innodb.test 2009-09-11 11:00:53 +0000 @@ -1317,6 +1317,10 @@ drop table t1; # Test for testable InnoDB status variables. This test # uses previous ones(pages_created, rows_deleted, ...). + +# InnoDB aligns the memory for the buffer pool to a page boundary. This may +# cause actual pool size to be one less than requested depending on exact +# alignment of obtained memory. --replace_result 512 511 SELECT variable_value FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_buffer_pool_pages_total'; SELECT variable_value FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_page_size'; === modified file 'sql/sql_select.cc' --- a/sql/sql_select.cc 2009-09-07 20:50:10 +0000 +++ b/sql/sql_select.cc 2009-09-11 11:00:53 +0000 @@ -3614,16 +3614,16 @@ add_ft_keys(DYNAMIC_ARRAY *keyuse_array, Item_func *arg0=(Item_func *)(func->arguments()[0]), *arg1=(Item_func *)(func->arguments()[1]); if (arg1->const_item() && - ((functype == Item_func::GE_FUNC && arg1->val_real() > 0) || - (functype == Item_func::GT_FUNC && arg1->val_real() >=0)) && arg0->type() == Item::FUNC_ITEM && - arg0->functype() == Item_func::FT_FUNC) + arg0->functype() == Item_func::FT_FUNC && + ((functype == Item_func::GE_FUNC && arg1->val_real() > 0) || + (functype == Item_func::GT_FUNC && arg1->val_real() >=0))) cond_func=(Item_func_match *) arg0; else if (arg0->const_item() && - ((functype == Item_func::LE_FUNC && arg0->val_real() > 0) || - (functype == Item_func::LT_FUNC && arg0->val_real() >=0)) && arg1->type() == Item::FUNC_ITEM && - arg1->functype() == Item_func::FT_FUNC) + arg1->functype() == Item_func::FT_FUNC && + ((functype == Item_func::LE_FUNC && arg0->val_real() > 0) || + (functype == Item_func::LT_FUNC && arg0->val_real() >=0))) cond_func=(Item_func_match *) arg1; } } === modified file 'support-files/build-tags' --- a/support-files/build-tags 2009-09-07 20:50:10 +0000 +++ b/support-files/build-tags 2009-09-11 11:00:53 +0000 @@ -4,7 +4,7 @@ rm -f TAGS filter='\.cc$\|\.c$\|\.h$\|\.yy$' list="find . -type f" -bzr root >/dev/null 2>/dev/null && list="bzr ls --from-root --kind=file --versioned" +bzr root >/dev/null 2>/dev/null && list="bzr ls --from-root -R --kind=file --versioned" $list |grep $filter |while read f; do

1 0

[Maria-developers] Rev 2726: Exclude innodb_plugin form MariaDB's max and max-no-ndb configurations and in file:///home/psergey/bzr-new/mysql-5.1-maria-contd3/
by Sergey Petrunya 10 Sep '09

10 Sep '09

At file:///home/psergey/bzr-new/mysql-5.1-maria-contd3/ ------------------------------------------------------------ revno: 2726 revision-id: psergey(a)askmonty.org-20090910081123-1hg7mxrfguchn811 parent: psergey(a)askmonty.org-20090910080750-tcyaxkh0wwpgqzad committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: mysql-5.1-maria-contd3 timestamp: Thu 2009-09-10 12:11:23 +0400 message: Exclude innodb_plugin form MariaDB's max and max-no-ndb configurations and from the default build. We have xtradb instead. === modified file 'BUILD/SETUP.sh' --- a/BUILD/SETUP.sh 2009-05-11 15:31:30 +0000 +++ b/BUILD/SETUP.sh 2009-09-10 08:11:23 +0000 @@ -153,7 +153,7 @@ base_configs="$base_configs --with-extra-charsets=complex " base_configs="$base_configs --enable-thread-safe-client " base_configs="$base_configs --with-big-tables" -base_configs="$base_configs --with-plugin-maria --with-maria-tmp-tables" +base_configs="$base_configs --with-plugin-maria --with-maria-tmp-tables --without-plugin-innodb_plugin" if test -d "$path/../cmd-line-utils/readline" then === modified file 'storage/innodb_plugin/plug.in' --- a/storage/innodb_plugin/plug.in 2009-07-30 12:42:56 +0000 +++ b/storage/innodb_plugin/plug.in 2009-09-10 08:11:23 +0000 @@ -15,7 +15,7 @@ # MYSQL_STORAGE_ENGINE(innodb_plugin,, [InnoDB Storage Engine], - [Transactional Tables using InnoDB], [max,max-no-ndb]) + [Transactional Tables using InnoDB],) MYSQL_PLUGIN_DIRECTORY(innodb_plugin, [storage/innodb_plugin]) MYSQL_PLUGIN_DYNAMIC(innodb_plugin, [ha_innodb_plugin.la]) MYSQL_PLUGIN_ACTIONS(innodb_plugin, [

2 2

[Maria-developers] Rev 2721: Merge xtradb-7 -> MariaDB in file:///home/psergey/bzr-new/mysql-5.1-maria-contd2/
by Sergey Petrunya 10 Sep '09

10 Sep '09

At file:///home/psergey/bzr-new/mysql-5.1-maria-contd2/ ------------------------------------------------------------ revno: 2721 revision-id: psergey(a)askmonty.org-20090908160458-fglojzacu8inqb9q parent: psergey(a)askmonty.org-20090907205010-tnvi4bomzq2b1eut parent: aleksandr.kuzminsky(a)percona.com-20090810223610-ssatw3bi8t3ggr5z committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: mysql-5.1-maria-contd2 timestamp: Tue 2009-09-08 20:04:58 +0400 message: Merge xtradb-7 -> MariaDB added: mysql-test.percona/ mysqltest-20081201061010-zymrrwrczns2vrex-27 mysql-test/patches/ patches-20081203050234-edoolglm28lyejuc-13 mysql-test/patches/events_stress.diff events_stress.diff-20090908160024-tii87qp66sk04a00-1 mysql-test/patches/information_schema.diff information_schema.d-20090908160027-1rc1imife37xem2d-1 mysql-test/patches/information_schema_db.diff information_schema_d-20090908160031-m7v1n1g41r1cfhif-1 mysql-test/patches/mysqlshow.diff mysqlshow.diff-20090908160036-0vxq2kivbnbugp6g-1 modified: README sp1f-readme-19700101030959-ipf4glwvob7zbr3norl5feyy3jwy3sod storage/xtradb/buf/buf0flu.c buf0flu.c-20081201061010-zymrrwrczns2vrex-50 storage/xtradb/handler/ha_innodb.cc ha_innodb.cc-20081201061010-zymrrwrczns2vrex-70 storage/xtradb/include/row0mysql.h row0mysql.h-20081201061010-zymrrwrczns2vrex-199 storage/xtradb/include/srv0srv.h srv0srv.h-20081201061010-zymrrwrczns2vrex-219 storage/xtradb/include/trx0trx.h trx0trx.h-20081201061010-zymrrwrczns2vrex-242 storage/xtradb/include/trx0trx.ic trx0trx.ic-20081201061010-zymrrwrczns2vrex-243 storage/xtradb/lock/lock0lock.c lock0lock.c-20081201061010-zymrrwrczns2vrex-269 storage/xtradb/row/row0mysql.c row0mysql.c-20081201061010-zymrrwrczns2vrex-336 storage/xtradb/row/row0sel.c row0sel.c-20081201061010-zymrrwrczns2vrex-339 storage/xtradb/srv/srv0srv.c srv0srv.c-20081201061010-zymrrwrczns2vrex-347 storage/xtradb/trx/trx0trx.c trx0trx.c-20081201061010-zymrrwrczns2vrex-359 ------------------------------------------------------------ revno: 0.6.2 revision-id: aleksandr.kuzminsky(a)percona.com-20090810223610-ssatw3bi8t3ggr5z parent: aleksandr.kuzminsky(a)percona.com-20090809070458-cjko9m35law9bx9r committer: Aleksandr Kuzminsky <aleksandr.kuzminsky(a)percona.com> branch nick: percona-xtradb timestamp: Mon 2009-08-10 22:36:10 +0000 message: sync with rev.94 modified: handler/ha_innodb.cc ha_innodb.cc-20081201061010-zymrrwrczns2vrex-70 include/srv0srv.h srv0srv.h-20081201061010-zymrrwrczns2vrex-219 srv/srv0srv.c srv0srv.c-20081201061010-zymrrwrczns2vrex-347 ------------------------------------------------------------ revno: 0.6.1 revision-id: aleksandr.kuzminsky(a)percona.com-20090809070458-cjko9m35law9bx9r parent: vadim(a)percona.com-20090626080424-i7bdmrag8piw0fc3 committer: Aleksandr Kuzminsky <aleksandr.kuzminsky(a)percona.com> branch nick: percona-xtradb timestamp: Sun 2009-08-09 07:04:58 +0000 message: sync with rev.93 modified: buf/buf0flu.c buf0flu.c-20081201061010-zymrrwrczns2vrex-50 buf/buf0rea.c buf0rea.c-20081201061010-zymrrwrczns2vrex-52 dict/dict0dict.c dict0dict.c-20081201061010-zymrrwrczns2vrex-57 handler/ha_innodb.cc ha_innodb.cc-20081201061010-zymrrwrczns2vrex-70 handler/innodb_patch_info.h innodb_patch_info.h-20081206234022-hep6ryfeacyr0572-1 include/row0mysql.h row0mysql.h-20081201061010-zymrrwrczns2vrex-199 include/srv0srv.h srv0srv.h-20081201061010-zymrrwrczns2vrex-219 include/trx0trx.h trx0trx.h-20081201061010-zymrrwrczns2vrex-242 include/trx0trx.ic trx0trx.ic-20081201061010-zymrrwrczns2vrex-243 include/univ.i univ.i-20081201061010-zymrrwrczns2vrex-248 lock/lock0lock.c lock0lock.c-20081201061010-zymrrwrczns2vrex-269 log/log0recv.c log0recv.c-20081201061010-zymrrwrczns2vrex-271 mysql-test/innodb-analyze.test innodbanalyze.test-20081203050234-edoolglm28lyejuc-6 mysql-test/innodb_xtradb_bug317074.result innodb_xtradb_bug317-20090326061054-ylrdb8libxw6u7e9-8 mysql-test/innodb_xtradb_bug317074.test innodb_xtradb_bug317-20090326061054-ylrdb8libxw6u7e9-9 mysql-test/patches/events_stress.diff events_stress.diff-20090420133118-3nkd032ighnz203p-1 mysql-test/patches/information_schema.diff information_schema.d-20081203050234-edoolglm28lyejuc-21 mysql-test/patches/information_schema_db.diff information_schema_d-20090420133118-3nkd032ighnz203p-2 mysql-test/patches/mysqlshow.diff mysqlshow.diff-20090420133118-3nkd032ighnz203p-6 row/row0mysql.c row0mysql.c-20081201061010-zymrrwrczns2vrex-336 row/row0sel.c row0sel.c-20081201061010-zymrrwrczns2vrex-339 srv/srv0srv.c srv0srv.c-20081201061010-zymrrwrczns2vrex-347 trx/trx0trx.c trx0trx.c-20081201061010-zymrrwrczns2vrex-359 Diff too large for email (1212 lines, the limit is 1000).

2 1

[Maria-developers] Rev 2723: MySQL 5.1 -> MariaDB 5.1 merge in file:///home/psergey/bzr-new/mysql-5.1-maria-contd2/
by Sergey Petrunya 10 Sep '09

10 Sep '09

At file:///home/psergey/bzr-new/mysql-5.1-maria-contd2/ ------------------------------------------------------------ revno: 2723 revision-id: psergey(a)askmonty.org-20090909210657-i1of5o03ymoozwvs parent: psergey(a)askmonty.org-20090908162138-wtd6r54dkcp4hg5a committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: mysql-5.1-maria-contd2 timestamp: Thu 2009-09-10 01:06:57 +0400 message: MySQL 5.1 -> MariaDB 5.1 merge - Post-merge fixes === modified file 'README' --- a/README 2009-09-08 16:04:58 +0000 +++ b/README 2009-09-09 21:06:57 +0000 @@ -1,47 +1,50 @@ -This is a release of MariaDB, a branch of MySQL. - -MariaDB is a drop-in replacement of MySQL, with more features, less -bugs and better performance. - -MariaDB is brought to you by many of the original developers of MySQL, -that now work for Monty Program Ab, and by many people in the -community. - -MySQL, which is the base of MariaDB, is brought to you by Sun. +This is a release of MySQL, a dual-license SQL database server. +MySQL is brought to you by the MySQL team at Sun Microsystems, Inc. License information can be found in these files: - For GPL (free) distributions, see the COPYING file and the EXCEPTIONS-CLIENT file. - -A description of the MariaDB project can be found at: -http://askmonty.org/wiki/index.php/MariaDB - -The differences between MariaDB and MySQL can be found at: -http://askmonty.org/wiki/index.php/MariaDB_versus_MySQL - -Documentation about MySQL can be found at: -http://dev.mysql.com/doc - -For further information about MySQL documentation, see: -- The current MySQL documentation: +- For commercial distributions, see the LICENSE.mysql file. + +GPLv2 Disclaimer +For the avoidance of doubt, except that if any license choice +other than GPL or LGPL is available it will apply instead, Sun +elects to use only the General Public License version 2 (GPLv2) +at this time for any software where a choice of GPL license versions +is made available with the language indicating that GPLv2 or any +later version may be used, or where a choice of which version of +the GPL is applied is otherwise unspecified. + +For further information about MySQL or additional documentation, see: +- The latest information about MySQL: http://www.mysql.com +- The current MySQL documentation: http://dev.mysql.com/doc Some manual sections of special interest: +- If you are migrating from an older version of MySQL, please read the + "Upgrading from..." section first! +- To see what MySQL can do, take a look at the features section. +- For installation instructions, see the Installing and Upgrading +chapter. +- For the new features/bugfix history, see the Change History appendix. +- For the currently known bugs/misfeatures (known errors) see the +Problems + and Common Errors appendix. - For a list of developers and other contributors, see the Credits appendix. A local copy of the MySQL Reference Manual can be found in the Docs directory in GNU Info format. You can also browse the manual online or -download it in any of several formats from -http://dev.mysql.com/doc +download it in any of several formats at the URL given earlier in this +file. ************************************************************ IMPORTANT: -Bug or error reports regarding MariaDB should be sent to -https://bugs.launchpad.net/maria -Bugs in the MySQL code can also be sent to http://bugs.mysql.com +Bug or error reports should be sent to http://bugs.mysql.com. + + *************************************************************************** %%The following software may be included in this product: === modified file 'mysql-test/r/innodb_mysql.result' --- a/mysql-test/r/innodb_mysql.result 2009-07-13 17:41:43 +0000 +++ b/mysql-test/r/innodb_mysql.result 2009-09-09 21:06:57 +0000 @@ -1746,7 +1746,7 @@ DROP TABLE t1; CREATE TABLE t1 (c1 DECIMAL(12,2), c2 DECIMAL(12,2), c3 DECIMAL(12,2), KEY (c3), KEY (c2, c3)) -ENGINE=InnoDB; +ENGINE=innodb; INSERT INTO t1 VALUES (1,1,1), (1,1,1), (1,1,2), (1,1,1), (1,1,2); SELECT 1 FROM (SELECT COUNT(DISTINCT c1) FROM t1 WHERE c2 IN (1, 1) AND c3 = 2 GROUP BY c2) x; === modified file 'mysql-test/suite/maria/r/maria3.result' --- a/mysql-test/suite/maria/r/maria3.result 2009-06-29 21:03:30 +0000 +++ b/mysql-test/suite/maria/r/maria3.result 2009-09-09 21:06:57 +0000 @@ -318,6 +318,7 @@ maria_sort_buffer_size 8388608 maria_stats_method nulls_unequal maria_sync_log_dir NEWFILE +maria_used_for_temp_tables ON show status like 'maria%'; Variable_name Value Maria_pagecache_blocks_not_flushed # === modified file 'sql/handler.cc' --- a/sql/handler.cc 2009-09-07 20:50:10 +0000 +++ b/sql/handler.cc 2009-09-09 21:06:57 +0000 @@ -63,7 +63,7 @@ const char *ha_row_type[] = { "", "FIXED", "DYNAMIC", "COMPRESSED", "REDUNDANT", "COMPACT", - /* Reserved to be "PAGE" in future versions */ "?", + "PAGE", "?","?","?" }; === modified file 'sql/opt_range.cc' --- a/sql/opt_range.cc 2009-09-07 20:50:10 +0000 +++ b/sql/opt_range.cc 2009-09-09 21:06:57 +0000 @@ -2243,7 +2243,7 @@ KEY *key_info; PARAM param; - if (check_stack_overrun(thd, 2*STACK_MIN_SIZE, buff)) + if (check_stack_overrun(thd, 2*STACK_MIN_SIZE + sizeof(PARAM), buff)) DBUG_RETURN(0); // Fatal error flag is set /* set up parameter that is passed to all functions */ @@ -3760,9 +3760,8 @@ DBUG_PRINT("info", ("index_merge scans cost %g", imerge_cost)); if (imerge_too_expensive || (imerge_cost > read_time) || - ((non_cpk_scan_records+cpk_scan_records >= - param->table->file->stats.records) && - read_time != DBL_MAX)) + ((non_cpk_scan_records+cpk_scan_records >= param->table->file->stats.records) && + read_time != DBL_MAX)) { /* Bail out if it is obvious that both index_merge and ROR-union will be @@ -4839,11 +4838,11 @@ { int idx; SEL_ARG **key,**end, **key_to_read= NULL; - ha_rows best_records; + ha_rows UNINIT_VAR(best_records); /* protected by key_to_read */ TRP_RANGE* read_plan= NULL; bool pk_is_clustered= param->table->file->primary_key_is_clustered(); DBUG_ENTER("get_key_scans_params"); - LINT_INIT(best_records); /* protected by key_to_read */ + /* Note that there may be trees that have type SEL_TREE::KEY but contain no key reads at all, e.g. tree for expression "key1 is not null" where key1 @@ -6440,13 +6439,6 @@ return 0; // Can't optimize this } - if ((key1->min_flag | key2->min_flag) & GEOM_FLAG) - { - key1->free_tree(); - key2->free_tree(); - return 0; // Can't optimize this - } - key1->use_count--; key2->use_count--; SEL_ARG *e1=key1->first(), *e2=key2->first(), *new_tree=0; @@ -6797,9 +6789,7 @@ SEL_ARG * SEL_ARG::insert(SEL_ARG *key) { - SEL_ARG *element,**par,*last_element; - LINT_INIT(par); - LINT_INIT(last_element); + SEL_ARG *element,**UNINIT_VAR(par),*UNINIT_VAR(last_element); for (element= this; element != &null_element ; ) { === modified file 'sql/sql_table.cc' --- a/sql/sql_table.cc 2009-09-07 20:50:10 +0000 +++ b/sql/sql_table.cc 2009-09-09 21:06:57 +0000 @@ -5951,6 +5951,8 @@ } if (!(used_fields & HA_CREATE_USED_KEY_BLOCK_SIZE)) create_info->key_block_size= table->s->key_block_size; + if (!(used_fields & HA_CREATE_USED_TRANSACTIONAL)) + create_info->transactional= table->s->transactional; if (!create_info->tablespace && create_info->storage_media != HA_SM_MEMORY) { === modified file 'sql/table.cc' --- a/sql/table.cc 2009-09-07 20:50:10 +0000 +++ b/sql/table.cc 2009-09-09 21:06:57 +0000 @@ -2491,11 +2491,8 @@ int4store(fileinfo+34,create_info->avg_row_length); fileinfo[38]= (create_info->default_table_charset ? create_info->default_table_charset->number : 0); - /* - In future versions, we will store in fileinfo[39] the values of the - TRANSACTIONAL and PAGE_CHECKSUM clauses of CREATE TABLE. - */ - fileinfo[39]= 0; + fileinfo[39]= (uchar) ((uint) create_info->transactional | + ((uint) create_info->page_checksum << 2)); fileinfo[40]= (uchar) create_info->row_type; /* Next few bytes where for RAID support */ fileinfo[41]= 0; === modified file 'storage/xtradb/handler/ha_innodb.cc' --- a/storage/xtradb/handler/ha_innodb.cc 2009-09-08 16:04:58 +0000 +++ b/storage/xtradb/handler/ha_innodb.cc 2009-09-09 21:06:57 +0000 @@ -9470,7 +9470,7 @@ There is no mechanism for updating InnoDB foreign key definitions. */ if (foreign_key_column_is_being_renamed(prebuilt, table)) { - return(COMPATIBLE_DATA_NO); + DBUG_RETURN(COMPATIBLE_DATA_NO); } /* Check that row format didn't change */

3 2

[Maria-developers] Rev 2725: Fix windows build: add strmov_overlapp.c to CMakeLists.txt in file:///home/psergey/bzr-new/mysql-5.1-maria-contd3/
by Sergey Petrunya 10 Sep '09

10 Sep '09

At file:///home/psergey/bzr-new/mysql-5.1-maria-contd3/ ------------------------------------------------------------ revno: 2725 revision-id: psergey(a)askmonty.org-20090910080750-tcyaxkh0wwpgqzad parent: psergey(a)askmonty.org-20090909215928-jd9v62cem7e2uukw committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: mysql-5.1-maria-contd3 timestamp: Thu 2009-09-10 12:07:50 +0400 message: Fix windows build: add strmov_overlapp.c to CMakeLists.txt === modified file 'libmysql/CMakeLists.txt' --- a/libmysql/CMakeLists.txt 2009-09-07 20:50:10 +0000 +++ b/libmysql/CMakeLists.txt 2009-09-10 08:07:50 +0000 @@ -94,7 +94,7 @@ ../mysys/safemalloc.c ../mysys/sha1.c ../strings/str2int.c ../strings/str_alloc.c ../strings/strcend.c ../strings/strcont.c ../strings/strend.c ../strings/strfill.c ../mysys/string.c ../strings/strinstr.c ../strings/strmake.c - ../strings/strmov.c ../strings/strnlen.c ../strings/strnmov.c ../strings/strtod.c + ../strings/strmov.c ../strings/strmov_overlapp.c ../strings/strnlen.c ../strings/strnmov.c ../strings/strtod.c ../strings/strtoll.c ../strings/strtoull.c ../strings/strxmov.c ../strings/strxnmov.c ../mysys/thr_mutex.c ../mysys/typelib.c ../vio/vio.c ../vio/viosocket.c ../vio/viossl.c ../vio/viosslfactories.c ../strings/xml.c ../mysys/mf_qsort.c === modified file 'strings/CMakeLists.txt' --- a/strings/CMakeLists.txt 2008-10-10 15:28:41 +0000 +++ b/strings/CMakeLists.txt 2009-09-10 08:07:50 +0000 @@ -23,7 +23,7 @@ ctype-latin1.c ctype-mb.c ctype-simple.c ctype-sjis.c ctype-tis620.c ctype-uca.c ctype-ucs2.c ctype-ujis.c ctype-utf8.c ctype-win1250ch.c ctype.c decimal.c int2str.c is_prefix.c llstr.c longlong2str.c my_strtoll10.c my_vsnprintf.c r_strinstr.c - str2int.c str_alloc.c strcend.c strend.c strfill.c strmake.c strmov.c strnmov.c + str2int.c str_alloc.c strcend.c strend.c strfill.c strmake.c strmov.c strmov_overlapp.c strnmov.c strtod.c strtol.c strtoll.c strtoul.c strtoull.c strxmov.c strxnmov.c xml.c my_strchr.c strcont.c strinstr.c strnlen.c strappend.c)

1 0

[Maria-developers] Rev 2726: Fix windows build: add strmov_overlapp.c to CMakeLists.txt in file:///home/psergey/bzr-new/mysql-5.1-maria-contd2/
by Sergey Petrunya 10 Sep '09

10 Sep '09

At file:///home/psergey/bzr-new/mysql-5.1-maria-contd2/ ------------------------------------------------------------ revno: 2726 revision-id: psergey(a)askmonty.org-20090909222555-h0o1cknpypnojuve parent: psergey(a)askmonty.org-20090909221538-1k8a5q1qzqdyymyw committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: mysql-5.1-maria-contd2 timestamp: Thu 2009-09-10 02:25:55 +0400 message: Fix windows build: add strmov_overlapp.c to CMakeLists.txt === modified file 'libmysql/CMakeLists.txt' --- a/libmysql/CMakeLists.txt 2009-09-07 20:50:10 +0000 +++ b/libmysql/CMakeLists.txt 2009-09-09 22:25:55 +0000 @@ -94,7 +94,7 @@ ../mysys/safemalloc.c ../mysys/sha1.c ../strings/str2int.c ../strings/str_alloc.c ../strings/strcend.c ../strings/strcont.c ../strings/strend.c ../strings/strfill.c ../mysys/string.c ../strings/strinstr.c ../strings/strmake.c - ../strings/strmov.c ../strings/strnlen.c ../strings/strnmov.c ../strings/strtod.c + ../strings/strmov.c ../strings/strmov_overlapp.c ../strings/strnlen.c ../strings/strnmov.c ../strings/strtod.c ../strings/strtoll.c ../strings/strtoull.c ../strings/strxmov.c ../strings/strxnmov.c ../mysys/thr_mutex.c ../mysys/typelib.c ../vio/vio.c ../vio/viosocket.c ../vio/viossl.c ../vio/viosslfactories.c ../strings/xml.c ../mysys/mf_qsort.c === modified file 'strings/CMakeLists.txt' --- a/strings/CMakeLists.txt 2008-10-10 15:28:41 +0000 +++ b/strings/CMakeLists.txt 2009-09-09 22:25:55 +0000 @@ -23,7 +23,7 @@ ctype-latin1.c ctype-mb.c ctype-simple.c ctype-sjis.c ctype-tis620.c ctype-uca.c ctype-ucs2.c ctype-ujis.c ctype-utf8.c ctype-win1250ch.c ctype.c decimal.c int2str.c is_prefix.c llstr.c longlong2str.c my_strtoll10.c my_vsnprintf.c r_strinstr.c - str2int.c str_alloc.c strcend.c strend.c strfill.c strmake.c strmov.c strnmov.c + str2int.c str_alloc.c strcend.c strend.c strfill.c strmake.c strmov.c strmov_overlapp.c strnmov.c strtod.c strtol.c strtoll.c strtoul.c strtoull.c strxmov.c strxnmov.c xml.c my_strchr.c strcont.c strinstr.c strnlen.c strappend.c)

1 0

[Maria-developers] Rev 2725: Apply Yasufumi's patch for MBUG#417751. in file:///home/psergey/bzr-new/mysql-5.1-maria-contd2/
by Sergey Petrunya 10 Sep '09

10 Sep '09

At file:///home/psergey/bzr-new/mysql-5.1-maria-contd2/ ------------------------------------------------------------ revno: 2725 revision-id: psergey(a)askmonty.org-20090909221538-1k8a5q1qzqdyymyw parent: psergey(a)askmonty.org-20090909215928-jd9v62cem7e2uukw committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: mysql-5.1-maria-contd2 timestamp: Thu 2009-09-10 02:15:38 +0400 message: Apply Yasufumi's patch for MBUG#417751. === modified file 'storage/xtradb/btr/btr0cur.c' --- a/storage/xtradb/btr/btr0cur.c 2009-06-25 01:43:25 +0000 +++ b/storage/xtradb/btr/btr0cur.c 2009-09-09 22:15:38 +0000 @@ -3267,6 +3267,7 @@ } while (rec != supremum) { + rec_t* next_rec; /* count recs */ if (stats_method == SRV_STATS_METHOD_IGNORE_NULLS) { n_recs++; @@ -3280,8 +3281,7 @@ n_not_nulls[j]++; } } - - rec_t* next_rec = page_rec_get_next(rec); + next_rec = page_rec_get_next(rec); if (next_rec == supremum) { break; } === modified file 'storage/xtradb/buf/buf0buddy.c' --- a/storage/xtradb/buf/buf0buddy.c 2009-06-25 01:43:25 +0000 +++ b/storage/xtradb/buf/buf0buddy.c 2009-09-09 22:15:38 +0000 @@ -461,16 +461,15 @@ actually is a properly initialized buf_page_t object. */ if (size >= PAGE_ZIP_MIN_SIZE) { - if (!have_page_hash_mutex) - mutex_exit(&zip_free_mutex); - /* This is a compressed page. */ mutex_t* mutex; if (!have_page_hash_mutex) { + mutex_exit(&zip_free_mutex); mutex_enter(&LRU_list_mutex); rw_lock_x_lock(&page_hash_latch); } + /* The src block may be split into smaller blocks, some of which may be free. Thus, the mach_read_from_4() calls below may attempt to read === modified file 'storage/xtradb/buf/buf0flu.c' --- a/storage/xtradb/buf/buf0flu.c 2009-08-09 07:04:58 +0000 +++ b/storage/xtradb/buf/buf0flu.c 2009-09-09 22:15:38 +0000 @@ -1233,13 +1233,13 @@ + BUF_FLUSH_EXTRA_MARGIN) && (distance < BUF_LRU_FREE_SEARCH_LEN)) { + mutex_t* block_mutex; if (!bpage->in_LRU_list) { /* reatart. but it is very optimistic */ bpage = UT_LIST_GET_LAST(buf_pool->LRU); continue; } - - mutex_t* block_mutex = buf_page_get_mutex(bpage); + block_mutex = buf_page_get_mutex_enter(bpage); retry_lock: mutex_enter(block_mutex); === modified file 'storage/xtradb/fil/fil0fil.c' --- a/storage/xtradb/fil/fil0fil.c 2009-06-25 01:43:25 +0000 +++ b/storage/xtradb/fil/fil0fil.c 2009-09-09 22:15:38 +0000 @@ -3039,6 +3039,13 @@ ulint i; int len; ib_uint64_t current_lsn; + ulint size_low, size_high, size; + ib_int64_t size_bytes; + dict_table_t* table; + dict_index_t* index; + fil_system_t* system; + fil_node_t* node = NULL; + fil_space_t* space; current_lsn = log_get_lsn(); @@ -3060,25 +3067,20 @@ success = os_file_write(filepath, file, page, 0, 0, UNIV_PAGE_SIZE); /* get file size */ - ulint size_low, size_high, size; - ib_int64_t size_bytes; os_file_get_size(file, &size_low, &size_high); size_bytes = (((ib_int64_t)size_high) << 32) + (ib_int64_t)size_low; /* get cruster index information */ - dict_table_t* table; - dict_index_t* index; table = dict_table_get_low(name); index = dict_table_get_first_index(table); ut_a(index->page==3); - /* read metadata from .exp file */ n_index = 0; - bzero(old_id, sizeof(old_id)); - bzero(new_id, sizeof(new_id)); - bzero(root_page, sizeof(root_page)); + memset(old_id, 0, sizeof(old_id)); + memset(new_id, 0, sizeof(new_id)); + memset(root_page, 0, sizeof(root_page)); info_file_path = fil_make_ibd_name(name, FALSE); len = strlen(info_file_path); @@ -3128,10 +3130,10 @@ mem_heap_t* heap = NULL; ulint offsets_[REC_OFFS_NORMAL_SIZE]; ulint* offsets = offsets_; + ib_int64_t offset; + size = (ulint) (size_bytes / UNIV_PAGE_SIZE); /* over write space id of all pages */ - ib_int64_t offset; - rec_offs_init(offsets_); fprintf(stderr, "InnoDB: Progress in %:"); @@ -3284,10 +3286,8 @@ } mem_free(info_file_path); - fil_system_t* system = fil_system; + system = fil_system; mutex_enter(&(system->mutex)); - fil_node_t* node = NULL; - fil_space_t* space; space = fil_space_get_by_id(id); if (space) node = UT_LIST_GET_FIRST(space->chain); === modified file 'storage/xtradb/handler/i_s.cc' --- a/storage/xtradb/handler/i_s.cc 2009-06-25 01:43:25 +0000 +++ b/storage/xtradb/handler/i_s.cc 2009-09-09 22:15:38 +0000 @@ -806,7 +806,7 @@ field_store_string(table->field[0], NULL); p = (char *)index->table_name; } - strcpy(table_name_raw, p); + strcpy(table_name_raw, (const char*)p); filename_to_tablename(table_name_raw, table_name, sizeof(table_name)); field_store_string(table->field[1], table_name); field_store_string(table->field[2], index->name); @@ -2861,7 +2861,7 @@ } else { rec_per_key = n_rows; } - snprintf(buff, 256, (i == index->n_uniq)?"%llu":"%llu, ", + ut_snprintf(buff, 256, (i == index->n_uniq)?"%llu":"%llu, ", rec_per_key); strncat(row_per_keys, buff, 256 - strlen(row_per_keys)); } === modified file 'storage/xtradb/srv/srv0srv.c' --- a/storage/xtradb/srv/srv0srv.c 2009-09-08 16:04:58 +0000 +++ b/storage/xtradb/srv/srv0srv.c 2009-09-09 22:15:38 +0000 @@ -2694,11 +2694,11 @@ /* defence line (max_checkpoint_age * 1/2) */ ib_uint64_t lsn = log_sys->lsn; - mutex_exit(&(log_sys->mutex)); - ib_uint64_t level, bpl; buf_page_t* bpage; + mutex_exit(&(log_sys->mutex)); + mutex_enter(&flush_list_mutex); level = 0; === modified file 'storage/xtradb/srv/srv0start.c' --- a/storage/xtradb/srv/srv0start.c 2009-08-03 20:09:53 +0000 +++ b/storage/xtradb/srv/srv0start.c 2009-09-09 22:15:38 +0000 @@ -1134,7 +1134,12 @@ os_aio_use_native_aio = FALSE; } else { /* On Win 2000 and XP use async i/o */ - os_aio_use_native_aio = TRUE; + //os_aio_use_native_aio = TRUE; + os_aio_use_native_aio = FALSE; + fprintf(stderr, + "InnoDB: Windows native async i/o is disabled as default.\n" + "InnoDB: It is not applicable for the current" + " multi io threads implementation.\n"); } #endif if (srv_file_flush_method_str == NULL) { @@ -1170,6 +1175,12 @@ } else if (0 == ut_strcmp(srv_file_flush_method_str, "async_unbuffered")) { srv_win_file_flush_method = SRV_WIN_IO_UNBUFFERED; + os_aio_use_native_aio = TRUE; + srv_n_read_io_threads = srv_n_write_io_threads = 1; + fprintf(stderr, + "InnoDB: 'async_unbuffered' was detected as innodb_flush_method.\n" + "InnoDB: Windows native async i/o is enabled.\n" + "InnoDB: And io threads are restricted.\n"); #endif } else { fprintf(stderr,

1 0

[Maria-developers] Rev 2724: Merge MySQL 5.1 -> MariaDB in file:///home/psergey/bzr-new/mysql-5.1-maria-contd2/
by Sergey Petrunya 09 Sep '09

09 Sep '09

At file:///home/psergey/bzr-new/mysql-5.1-maria-contd2/ ------------------------------------------------------------ revno: 2724 revision-id: psergey(a)askmonty.org-20090909215928-jd9v62cem7e2uukw parent: psergey(a)askmonty.org-20090909210657-i1of5o03ymoozwvs committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: mysql-5.1-maria-contd2 timestamp: Thu 2009-09-10 01:59:28 +0400 message: Merge MySQL 5.1 -> MariaDB - Don't follow mainline on use of UNINIT_VAR yet, we haven't merged it in. === modified file 'sql/opt_range.cc' --- a/sql/opt_range.cc 2009-09-09 21:06:57 +0000 +++ b/sql/opt_range.cc 2009-09-09 21:59:28 +0000 @@ -2243,7 +2243,7 @@ KEY *key_info; PARAM param; - if (check_stack_overrun(thd, 2*STACK_MIN_SIZE + sizeof(PARAM), buff)) + if (check_stack_overrun(thd, 2*STACK_MIN_SIZE, buff)) DBUG_RETURN(0); // Fatal error flag is set /* set up parameter that is passed to all functions */ @@ -3760,8 +3760,9 @@ DBUG_PRINT("info", ("index_merge scans cost %g", imerge_cost)); if (imerge_too_expensive || (imerge_cost > read_time) || - ((non_cpk_scan_records+cpk_scan_records >= param->table->file->stats.records) && - read_time != DBL_MAX)) + ((non_cpk_scan_records+cpk_scan_records >= + param->table->file->stats.records) && + read_time != DBL_MAX)) { /* Bail out if it is obvious that both index_merge and ROR-union will be @@ -4838,11 +4839,11 @@ { int idx; SEL_ARG **key,**end, **key_to_read= NULL; - ha_rows UNINIT_VAR(best_records); /* protected by key_to_read */ + ha_rows best_records; TRP_RANGE* read_plan= NULL; bool pk_is_clustered= param->table->file->primary_key_is_clustered(); DBUG_ENTER("get_key_scans_params"); - + LINT_INIT(best_records); /* protected by key_to_read */ /* Note that there may be trees that have type SEL_TREE::KEY but contain no key reads at all, e.g. tree for expression "key1 is not null" where key1 @@ -6439,6 +6440,13 @@ return 0; // Can't optimize this } + if ((key1->min_flag | key2->min_flag) & GEOM_FLAG) + { + key1->free_tree(); + key2->free_tree(); + return 0; // Can't optimize this + } + key1->use_count--; key2->use_count--; SEL_ARG *e1=key1->first(), *e2=key2->first(), *new_tree=0; @@ -6789,7 +6797,9 @@ SEL_ARG * SEL_ARG::insert(SEL_ARG *key) { - SEL_ARG *element,**UNINIT_VAR(par),*UNINIT_VAR(last_element); + SEL_ARG *element,**par,*last_element; + LINT_INIT(par); + LINT_INIT(last_element); for (element= this; element != &null_element ; ) {

1 0

[Maria-developers] Rev 2863: Re-commit BUG#42742 in file:///home/psergey/dev/mysql-next-subqbugs/
by Sergey Petrunya 09 Sep '09

09 Sep '09

At file:///home/psergey/dev/mysql-next-subqbugs/ ------------------------------------------------------------ revno: 2863 revision-id: psergey(a)askmonty.org-20090909185418-aqe8vozjy1zo037q parent: alik(a)sun.com-20090828100112-r73xkx8dhekz5bbb committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: mysql-next-subqbugs timestamp: Wed 2009-09-09 22:54:18 +0400 message: Re-commit BUG#42742 === modified file 'mysql-test/r/subselect_sj2.result' --- a/mysql-test/r/subselect_sj2.result 2009-03-21 15:31:38 +0000 +++ b/mysql-test/r/subselect_sj2.result 2009-09-09 18:54:18 +0000 @@ -689,3 +689,19 @@ id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables drop table t1, t2; +# +# BUG#42742: crash in setup_sj_materialization, Copy_field::set +# +create table t3 ( c1 year) engine=innodb; +insert into t3 values (2135),(2142); +create table t2 (c1 tinytext,c2 text,c6 timestamp) engine=innodb; +# The following must not crash, EXPLAIN should show one SJ strategy, not a mix: +explain select 1 from t2 where +c2 in (select 1 from t3, t2) and +c1 in (select convert(c6,char(1)) from t2); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 1 Using where +1 PRIMARY t2 ALL NULL NULL NULL NULL 1 +1 PRIMARY t2 ALL NULL NULL NULL NULL 1 Using where; Using join buffer +1 PRIMARY t3 ALL NULL NULL NULL NULL 2 FirstMatch(t2); Using join buffer +drop table t2, t3; === modified file 'mysql-test/r/subselect_sj2_jcl6.result' --- a/mysql-test/r/subselect_sj2_jcl6.result 2009-06-19 09:12:06 +0000 +++ b/mysql-test/r/subselect_sj2_jcl6.result 2009-09-09 18:54:18 +0000 @@ -693,6 +693,22 @@ id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables drop table t1, t2; +# +# BUG#42742: crash in setup_sj_materialization, Copy_field::set +# +create table t3 ( c1 year) engine=innodb; +insert into t3 values (2135),(2142); +create table t2 (c1 tinytext,c2 text,c6 timestamp) engine=innodb; +# The following must not crash, EXPLAIN should show one SJ strategy, not a mix: +explain select 1 from t2 where +c2 in (select 1 from t3, t2) and +c1 in (select convert(c6,char(1)) from t2); +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t2 ALL NULL NULL NULL NULL 1 Using where +1 PRIMARY t2 ALL NULL NULL NULL NULL 1 Using join buffer +1 PRIMARY t2 ALL NULL NULL NULL NULL 1 Using where; Using join buffer +1 PRIMARY t3 ALL NULL NULL NULL NULL 2 FirstMatch(t2); Using join buffer +drop table t2, t3; set join_cache_level=default; show variables like 'join_cache_level'; Variable_name Value === modified file 'mysql-test/t/subselect_sj2.test' --- a/mysql-test/t/subselect_sj2.test 2009-03-21 15:31:38 +0000 +++ b/mysql-test/t/subselect_sj2.test 2009-09-09 18:54:18 +0000 @@ -872,3 +872,15 @@ explain select 1 from t2 where c2 = any (select log10(null) from t1 where c6 <null) ; drop table t1, t2; +--echo # +--echo # BUG#42742: crash in setup_sj_materialization, Copy_field::set +--echo # +create table t3 ( c1 year) engine=innodb; +insert into t3 values (2135),(2142); +create table t2 (c1 tinytext,c2 text,c6 timestamp) engine=innodb; +-- echo # The following must not crash, EXPLAIN should show one SJ strategy, not a mix: +explain select 1 from t2 where + c2 in (select 1 from t3, t2) and + c1 in (select convert(c6,char(1)) from t2); +drop table t2, t3; + === modified file 'sql/sql_select.cc' --- a/sql/sql_select.cc 2009-07-28 14:16:37 +0000 +++ b/sql/sql_select.cc 2009-09-09 18:54:18 +0000 @@ -8006,7 +8006,11 @@ uint i_end= first + join->best_positions[first].n_sj_tables; for (uint i= first; i < i_end; i++) + { + if (i != first) + join->best_positions[i].sj_strategy= SJ_OPT_NONE; handled_tabs |= join->best_positions[i].table->table->map; + } if (tablenr != first) pos->sj_strategy= SJ_OPT_NONE;

1 0

[Maria-developers] [Branch ~maria-captains/maria/5.1] Rev 2731: Added a missing dependency to build maria lib
by noreply＠launchpad.net 09 Sep '09

09 Sep '09

------------------------------------------------------------ revno: 2731 committer: Igor Babaev <igor(a)mysql.com> branch nick: maria-5.1-fix-windows timestamp: Wed 2009-09-09 11:11:31 -0700 message: Added a missing dependency to build maria lib modified: storage/maria/CMakeLists.txt -- lp:maria https://code.launchpad.net/~maria-captains/maria/5.1 Your team Maria developers is subscribed to branch lp:maria. To unsubscribe from this branch go to https://code.launchpad.net/~maria-captains/maria/5.1/+edit-subscription.

1 0

[Maria-developers] Progress (by Knielsen): mysqlbinlog: remove undesired effects of format description binlog statement (50)
by worklog-noreply＠askmonty.org 09 Sep '09

09 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: mysqlbinlog: remove undesired effects of format description binlog statement CREATION DATE..: Mon, 17 Aug 2009, 14:12 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 50 (http://askmonty.org/worklog/?tid=50) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 10 ESTIMATE.......: 10 (hours remain) ORIG. ESTIMATE.: 10 PROGRESS NOTES: -=-=(Knielsen - Wed, 09 Sep 2009, 18:31)=-=- Discussed with MySQL developers on Bug#46640. Committed a suggested fix: https://lists.launchpad.net/maria-developers/msg00926.html Worked 8 hours and estimate 10 hours remain (original estimate increased by 18 hours). -=-=(Guest - Wed, 09 Sep 2009, 11:50)=-=- High Level Description modified. --- /tmp/wklog.50.old.11584 2009-09-09 11:50:12.000000000 +0300 +++ /tmp/wklog.50.new.11584 2009-09-09 11:50:12.000000000 +0300 @@ -5,7 +5,11 @@ This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. -2. When one applies "format description binlog statement" at the slave, it -will change the slave's server_id when applied. +2. When one executes a BINLOG statement (containing "format description binlog +statement" or other events), this will cause subsequent statements run in +the same session to be binlogged with the server id of the last event +executed, not with the real server id. This can cause problems like infinite +recursive application in a circular replication topology. This WL is to fix these issues. + -=-=(Knielsen - Tue, 08 Sep 2009, 15:07)=-=- Low Level Design modified. --- /tmp/wklog.50.old.19208 2009-09-08 15:07:09.000000000 +0300 +++ /tmp/wklog.50.new.19208 2009-09-08 15:07:09.000000000 +0300 @@ -1 +1,34 @@ +I think the fix for point 2 is to replace the call to +apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct +call to ev->apply_event(). However, I need to check two things to be sure this +is correct: + +1. The existing code does + + if (!ev->when) ev->when= my_time(0) + +Need to understand if this is needed/correct or not. + +2. The existing code does ev->update_pos(). I think this is redundant for +BINLOG statement (as it uses a fake rli structure), but I need to check to +make sure. + +Once this is done, point 1 may no longer be needed. The user can use +--base64-output=never when applying the output to a 5.0 server, and omit that +option when applying the output to a 5.1 server. There should be no need to +omit the format description event in the output when other BINLOG statements +for row events are present, as it no longer changes the server id. In fact the +format description event is required to be able to execute other BINLOG +statements, as its purpose is to define the binary format of the events +contained in these statements. + +Alternatively, we could implement that if the format description event of the +source binlog has server version < 5.1, and --base64-output=auto (default), +then the format description event is omitted (and should any BINLOG statement +be needed, unlikely as it is from a 5.0 server, we will need to throw an +error). + +The binlog format version for 5.0 and 5.1 is actually the same, hence the need +to look at server version to guess if the format description event can be +omitted. -=-=(Knielsen - Tue, 08 Sep 2009, 14:29)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.17531 2009-09-08 14:29:30.000000000 +0300 +++ /tmp/wklog.50.new.17531 2009-09-08 14:29:30.000000000 +0300 @@ -12,3 +12,37 @@ One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? +First, the problem is not the format description log event. In fact all log +events have their own value of server_id, and every event in the BINLOG +statement is executed with its own id as server_id. + +It seems it was introduced deliberately in the patch for Bug#32407. This patch +makes sql thread and BINLOG statement share code path for executing the binary +event, even though the bug is really about something different (outputting +format description event to allow proper execution of other BINLOG +statements). + +Nevertheless, I think using the server_id in the events of a BINLOG statement +is wrong: + +1. It is different behaviour from executing normal statement-based events not +written using BINLOG. + +2. It causes the possibility for infinite cycle of replication in a cyclic +replication setup, if the server ids in BINLOG are different from all other +server ids in the cycle. + +3. The functionality of applying events with original server id from the event +is still available by pointing the slave thread to the binlog. The +functionality of applying row-based binlog events with server id of the server +executing the BINLOG statements is not otherwise available. + +In fact most of the code from the slave thread that is now also run when +executing BINLOG statements is redundant, or does the wrong thing (like +updating binlog position). + +Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be +able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is +it to be able to simulate the effect of the slave sql thread? I think the +former is the most important one. + -=-=(Knielsen - Mon, 07 Sep 2009, 10:06)=-=- Research worklog, bugs, and existing server code. Worked 2 hours and estimate 0 hours remain (original estimate increased by 2 hours). -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- Dependency created: 39 now depends on 50 -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.11389 2009-08-17 14:13:05.000000000 +0300 +++ /tmp/wklog.50.new.11389 2009-08-17 14:13:05.000000000 +0300 @@ -1 +1,14 @@ +First item +---------- +AFAIU what needs to be done is: +1. record a source server version (it is in the first binlog event). +2. don't emit a BINLOG statement if the recorded version number is 5.0.x or + below. + +and we'll get a 5.0-applicable binlog. + +Second item +----------- +One question that one needs to sort out before disabling server_id change is +why it was put there in the first place? Should it be always removed? DESCRIPTION: According to complaints in BUG#46640: 1. mysqlbinlog will emit a 5.0-incompatible "format description binlog statement" even when reading a binary log that was produced by 5.0. This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. 2. When one executes a BINLOG statement (containing "format description binlog statement" or other events), this will cause subsequent statements run in the same session to be binlogged with the server id of the last event executed, not with the real server id. This can cause problems like infinite recursive application in a circular replication topology. This WL is to fix these issues. HIGH-LEVEL SPECIFICATION: First item ---------- AFAIU what needs to be done is: 1. record a source server version (it is in the first binlog event). 2. don't emit a BINLOG statement if the recorded version number is 5.0.x or below. and we'll get a 5.0-applicable binlog. Second item ----------- One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? First, the problem is not the format description log event. In fact all log events have their own value of server_id, and every event in the BINLOG statement is executed with its own id as server_id. It seems it was introduced deliberately in the patch for Bug#32407. This patch makes sql thread and BINLOG statement share code path for executing the binary event, even though the bug is really about something different (outputting format description event to allow proper execution of other BINLOG statements). Nevertheless, I think using the server_id in the events of a BINLOG statement is wrong: 1. It is different behaviour from executing normal statement-based events not written using BINLOG. 2. It causes the possibility for infinite cycle of replication in a cyclic replication setup, if the server ids in BINLOG are different from all other server ids in the cycle. 3. The functionality of applying events with original server id from the event is still available by pointing the slave thread to the binlog. The functionality of applying row-based binlog events with server id of the server executing the BINLOG statements is not otherwise available. In fact most of the code from the slave thread that is now also run when executing BINLOG statements is redundant, or does the wrong thing (like updating binlog position). Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is it to be able to simulate the effect of the slave sql thread? I think the former is the most important one. LOW-LEVEL DESIGN: I think the fix for point 2 is to replace the call to apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct call to ev->apply_event(). However, I need to check two things to be sure this is correct: 1. The existing code does if (!ev->when) ev->when= my_time(0) Need to understand if this is needed/correct or not. 2. The existing code does ev->update_pos(). I think this is redundant for BINLOG statement (as it uses a fake rli structure), but I need to check to make sure. Once this is done, point 1 may no longer be needed. The user can use --base64-output=never when applying the output to a 5.0 server, and omit that option when applying the output to a 5.1 server. There should be no need to omit the format description event in the output when other BINLOG statements for row events are present, as it no longer changes the server id. In fact the format description event is required to be able to execute other BINLOG statements, as its purpose is to define the binary format of the events contained in these statements. Alternatively, we could implement that if the format description event of the source binlog has server version < 5.1, and --base64-output=auto (default), then the format description event is omitted (and should any BINLOG statement be needed, unlikely as it is from a 5.0 server, we will need to throw an error). The binlog format version for 5.0 and 5.1 is actually the same, hence the need to look at server version to guess if the format description event can be omitted. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Progress (by Knielsen): mysqlbinlog: remove undesired effects of format description binlog statement (50)
by worklog-noreply＠askmonty.org 09 Sep '09

09 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: mysqlbinlog: remove undesired effects of format description binlog statement CREATION DATE..: Mon, 17 Aug 2009, 14:12 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 50 (http://askmonty.org/worklog/?tid=50) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 10 ESTIMATE.......: 10 (hours remain) ORIG. ESTIMATE.: 10 PROGRESS NOTES: -=-=(Knielsen - Wed, 09 Sep 2009, 18:31)=-=- Discussed with MySQL developers on Bug#46640. Committed a suggested fix: https://lists.launchpad.net/maria-developers/msg00926.html Worked 8 hours and estimate 10 hours remain (original estimate increased by 18 hours). -=-=(Guest - Wed, 09 Sep 2009, 11:50)=-=- High Level Description modified. --- /tmp/wklog.50.old.11584 2009-09-09 11:50:12.000000000 +0300 +++ /tmp/wklog.50.new.11584 2009-09-09 11:50:12.000000000 +0300 @@ -5,7 +5,11 @@ This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. -2. When one applies "format description binlog statement" at the slave, it -will change the slave's server_id when applied. +2. When one executes a BINLOG statement (containing "format description binlog +statement" or other events), this will cause subsequent statements run in +the same session to be binlogged with the server id of the last event +executed, not with the real server id. This can cause problems like infinite +recursive application in a circular replication topology. This WL is to fix these issues. + -=-=(Knielsen - Tue, 08 Sep 2009, 15:07)=-=- Low Level Design modified. --- /tmp/wklog.50.old.19208 2009-09-08 15:07:09.000000000 +0300 +++ /tmp/wklog.50.new.19208 2009-09-08 15:07:09.000000000 +0300 @@ -1 +1,34 @@ +I think the fix for point 2 is to replace the call to +apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct +call to ev->apply_event(). However, I need to check two things to be sure this +is correct: + +1. The existing code does + + if (!ev->when) ev->when= my_time(0) + +Need to understand if this is needed/correct or not. + +2. The existing code does ev->update_pos(). I think this is redundant for +BINLOG statement (as it uses a fake rli structure), but I need to check to +make sure. + +Once this is done, point 1 may no longer be needed. The user can use +--base64-output=never when applying the output to a 5.0 server, and omit that +option when applying the output to a 5.1 server. There should be no need to +omit the format description event in the output when other BINLOG statements +for row events are present, as it no longer changes the server id. In fact the +format description event is required to be able to execute other BINLOG +statements, as its purpose is to define the binary format of the events +contained in these statements. + +Alternatively, we could implement that if the format description event of the +source binlog has server version < 5.1, and --base64-output=auto (default), +then the format description event is omitted (and should any BINLOG statement +be needed, unlikely as it is from a 5.0 server, we will need to throw an +error). + +The binlog format version for 5.0 and 5.1 is actually the same, hence the need +to look at server version to guess if the format description event can be +omitted. -=-=(Knielsen - Tue, 08 Sep 2009, 14:29)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.17531 2009-09-08 14:29:30.000000000 +0300 +++ /tmp/wklog.50.new.17531 2009-09-08 14:29:30.000000000 +0300 @@ -12,3 +12,37 @@ One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? +First, the problem is not the format description log event. In fact all log +events have their own value of server_id, and every event in the BINLOG +statement is executed with its own id as server_id. + +It seems it was introduced deliberately in the patch for Bug#32407. This patch +makes sql thread and BINLOG statement share code path for executing the binary +event, even though the bug is really about something different (outputting +format description event to allow proper execution of other BINLOG +statements). + +Nevertheless, I think using the server_id in the events of a BINLOG statement +is wrong: + +1. It is different behaviour from executing normal statement-based events not +written using BINLOG. + +2. It causes the possibility for infinite cycle of replication in a cyclic +replication setup, if the server ids in BINLOG are different from all other +server ids in the cycle. + +3. The functionality of applying events with original server id from the event +is still available by pointing the slave thread to the binlog. The +functionality of applying row-based binlog events with server id of the server +executing the BINLOG statements is not otherwise available. + +In fact most of the code from the slave thread that is now also run when +executing BINLOG statements is redundant, or does the wrong thing (like +updating binlog position). + +Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be +able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is +it to be able to simulate the effect of the slave sql thread? I think the +former is the most important one. + -=-=(Knielsen - Mon, 07 Sep 2009, 10:06)=-=- Research worklog, bugs, and existing server code. Worked 2 hours and estimate 0 hours remain (original estimate increased by 2 hours). -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- Dependency created: 39 now depends on 50 -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.11389 2009-08-17 14:13:05.000000000 +0300 +++ /tmp/wklog.50.new.11389 2009-08-17 14:13:05.000000000 +0300 @@ -1 +1,14 @@ +First item +---------- +AFAIU what needs to be done is: +1. record a source server version (it is in the first binlog event). +2. don't emit a BINLOG statement if the recorded version number is 5.0.x or + below. + +and we'll get a 5.0-applicable binlog. + +Second item +----------- +One question that one needs to sort out before disabling server_id change is +why it was put there in the first place? Should it be always removed? DESCRIPTION: According to complaints in BUG#46640: 1. mysqlbinlog will emit a 5.0-incompatible "format description binlog statement" even when reading a binary log that was produced by 5.0. This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. 2. When one executes a BINLOG statement (containing "format description binlog statement" or other events), this will cause subsequent statements run in the same session to be binlogged with the server id of the last event executed, not with the real server id. This can cause problems like infinite recursive application in a circular replication topology. This WL is to fix these issues. HIGH-LEVEL SPECIFICATION: First item ---------- AFAIU what needs to be done is: 1. record a source server version (it is in the first binlog event). 2. don't emit a BINLOG statement if the recorded version number is 5.0.x or below. and we'll get a 5.0-applicable binlog. Second item ----------- One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? First, the problem is not the format description log event. In fact all log events have their own value of server_id, and every event in the BINLOG statement is executed with its own id as server_id. It seems it was introduced deliberately in the patch for Bug#32407. This patch makes sql thread and BINLOG statement share code path for executing the binary event, even though the bug is really about something different (outputting format description event to allow proper execution of other BINLOG statements). Nevertheless, I think using the server_id in the events of a BINLOG statement is wrong: 1. It is different behaviour from executing normal statement-based events not written using BINLOG. 2. It causes the possibility for infinite cycle of replication in a cyclic replication setup, if the server ids in BINLOG are different from all other server ids in the cycle. 3. The functionality of applying events with original server id from the event is still available by pointing the slave thread to the binlog. The functionality of applying row-based binlog events with server id of the server executing the BINLOG statements is not otherwise available. In fact most of the code from the slave thread that is now also run when executing BINLOG statements is redundant, or does the wrong thing (like updating binlog position). Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is it to be able to simulate the effect of the slave sql thread? I think the former is the most important one. LOW-LEVEL DESIGN: I think the fix for point 2 is to replace the call to apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct call to ev->apply_event(). However, I need to check two things to be sure this is correct: 1. The existing code does if (!ev->when) ev->when= my_time(0) Need to understand if this is needed/correct or not. 2. The existing code does ev->update_pos(). I think this is redundant for BINLOG statement (as it uses a fake rli structure), but I need to check to make sure. Once this is done, point 1 may no longer be needed. The user can use --base64-output=never when applying the output to a 5.0 server, and omit that option when applying the output to a 5.1 server. There should be no need to omit the format description event in the output when other BINLOG statements for row events are present, as it no longer changes the server id. In fact the format description event is required to be able to execute other BINLOG statements, as its purpose is to define the binary format of the events contained in these statements. Alternatively, we could implement that if the format description event of the source binlog has server version < 5.1, and --base64-output=auto (default), then the format description event is omitted (and should any BINLOG statement be needed, unlikely as it is from a 5.0 server, we will need to throw an error). The binlog format version for 5.0 and 5.1 is actually the same, hence the need to look at server version to guess if the format description event can be omitted. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2731) Bug#46640
by knielsen＠knielsen-hq.org 09 Sep '09

09 Sep '09

#At lp:maria 2731 knielsen(a)knielsen-hq.org 2009-09-09 Bug#46640: output from mysqlbinlog command in 5.1 breaks replication The BINLOG statement was sharing too much code with the slave SQL thread, introduced with the patch for Bug#32407. This caused statements to be logged with the wrong server_id, the id stored inside the events of the BINLOG statement rather than the id of the running server. Fix by rearranging code a bit so that only relevant parts of the code are executed by the BINLOG statement. modified: mysql-test/extra/binlog_tests/binlog.test mysql-test/suite/binlog/r/binlog_row_binlog.result mysql-test/suite/binlog/r/binlog_stm_binlog.result sql/log_event.cc sql/slave.cc sql/slave.h sql/sql_binlog.cc per-file messages: mysql-test/extra/binlog_tests/binlog.test Add test case. mysql-test/suite/binlog/r/binlog_row_binlog.result Add test case. mysql-test/suite/binlog/r/binlog_stm_binlog.result Add test case. sql/log_event.cc Move rows_event_stmt_clean() call from update_pos() to apply_event(). This in any case makes more sense, and is needed as update_pos() is no longer called when executing BINLOG statements. sql/slave.cc The skip flag is no longer needed, as the code path for BINLOG statement has been cleaned up. sql/slave.h The skip flag is no longer needed, as the code path for BINLOG statement has been cleaned up. sql/sql_binlog.cc Don't invoke the update_pos() code path for the BINLOG statement, as it contains code that is redundant and/or harmful (especially setting thd->server_id). === modified file 'mysql-test/extra/binlog_tests/binlog.test' --- a/mysql-test/extra/binlog_tests/binlog.test 2009-03-25 19:41:16 +0000 +++ b/mysql-test/extra/binlog_tests/binlog.test 2009-09-09 15:27:23 +0000 @@ -258,3 +258,39 @@ dec $it; } show master status /* must show new binlog index after rotating */; drop table t3; + + +# +# Bug#46640: output from mysqlbinlog command in 5.1 breaks replication +# + +connect (fresh,localhost,root,,test); +connection fresh; + +RESET MASTER; +CREATE TABLE t1 (a INT PRIMARY KEY); + +# Format description event, with server_id = 10; +BINLOG ' +3u9kSA8KAAAAZgAAAGoAAAABAAQANS4xLjM1LW1hcmlhLWJldGExLWRlYnVnLWxvZwAAAAAAAAAA +AAAAAAAAAAAAAAAAAADe72RIEzgNAAgAEgAEBAQEEgAAUwAEGggAAAAICAgC +'; + +# What server_id is logged for a statement? Should be our own, not the +# one from the format description event. +INSERT INTO t1 VALUES (1); + +# INSERT INTO t1 VALUES (2), with server_id=20. Check that this is logged +# with our own server id, not the 20 from the BINLOG statement. +BINLOG ' +3u9kSBMUAAAAKQAAAJEBAAAAABoAAAAAAAAABHRlc3QAAnQxAAEDAAA= +3u9kSBcUAAAAIgAAALMBAAAQABoAAAAAAAEAAf/+AgAAAA== +'; + +# Show binlog events to check that server ids are correct. +--replace_column 1 # 2 # 5 # +--replace_regex /Server ver: .*, Binlog ver: .*/Server ver: #, Binlog ver: #/ /table_id: [0-9]+/table_id: #/ +SHOW BINLOG EVENTS; + +DROP TABLE t1; +disconnect fresh; === modified file 'mysql-test/suite/binlog/r/binlog_row_binlog.result' --- a/mysql-test/suite/binlog/r/binlog_row_binlog.result 2009-03-25 19:41:16 +0000 +++ b/mysql-test/suite/binlog/r/binlog_row_binlog.result 2009-09-09 15:27:23 +0000 @@ -1298,3 +1298,27 @@ show master status /* must show new binl File Position Binlog_Do_DB Binlog_Ignore_DB master-bin.000002 106 drop table t3; +RESET MASTER; +CREATE TABLE t1 (a INT PRIMARY KEY); +BINLOG ' +3u9kSA8KAAAAZgAAAGoAAAABAAQANS4xLjM1LW1hcmlhLWJldGExLWRlYnVnLWxvZwAAAAAAAAAA +AAAAAAAAAAAAAAAAAADe72RIEzgNAAgAEgAEBAQEEgAAUwAEGggAAAAICAgC +'; +INSERT INTO t1 VALUES (1); +BINLOG ' +3u9kSBMUAAAAKQAAAJEBAAAAABoAAAAAAAAABHRlc3QAAnQxAAEDAAA= +3u9kSBcUAAAAIgAAALMBAAAQABoAAAAAAAEAAf/+AgAAAA== +'; +SHOW BINLOG EVENTS; +Log_name Pos Event_type Server_id End_log_pos Info +# # Format_desc 1 # Server ver: #, Binlog ver: # +# # Query 1 # use `test`; CREATE TABLE t1 (a INT PRIMARY KEY) +# # Query 1 # use `test`; BEGIN +# # Table_map 1 # table_id: # (test.t1) +# # Write_rows 1 # table_id: # flags: STMT_END_F +# # Query 1 # use `test`; COMMIT +# # Query 1 # use `test`; BEGIN +# # Table_map 1 # table_id: # (test.t1) +# # Write_rows 1 # table_id: # flags: STMT_END_F +# # Query 1 # use `test`; COMMIT +DROP TABLE t1; === modified file 'mysql-test/suite/binlog/r/binlog_stm_binlog.result' --- a/mysql-test/suite/binlog/r/binlog_stm_binlog.result 2009-03-25 19:41:16 +0000 +++ b/mysql-test/suite/binlog/r/binlog_stm_binlog.result 2009-09-09 15:27:23 +0000 @@ -773,3 +773,24 @@ show master status /* must show new binl File Position Binlog_Do_DB Binlog_Ignore_DB master-bin.000002 106 drop table t3; +RESET MASTER; +CREATE TABLE t1 (a INT PRIMARY KEY); +BINLOG ' +3u9kSA8KAAAAZgAAAGoAAAABAAQANS4xLjM1LW1hcmlhLWJldGExLWRlYnVnLWxvZwAAAAAAAAAA +AAAAAAAAAAAAAAAAAADe72RIEzgNAAgAEgAEBAQEEgAAUwAEGggAAAAICAgC +'; +INSERT INTO t1 VALUES (1); +BINLOG ' +3u9kSBMUAAAAKQAAAJEBAAAAABoAAAAAAAAABHRlc3QAAnQxAAEDAAA= +3u9kSBcUAAAAIgAAALMBAAAQABoAAAAAAAEAAf/+AgAAAA== +'; +SHOW BINLOG EVENTS; +Log_name Pos Event_type Server_id End_log_pos Info +# # Format_desc 1 # Server ver: #, Binlog ver: # +# # Query 1 # use `test`; CREATE TABLE t1 (a INT PRIMARY KEY) +# # Query 1 # use `test`; INSERT INTO t1 VALUES (1) +# # Query 1 # use `test`; BEGIN +# # Table_map 1 # table_id: # (test.t1) +# # Write_rows 1 # table_id: # flags: STMT_END_F +# # Query 1 # use `test`; COMMIT +DROP TABLE t1; === modified file 'sql/log_event.cc' --- a/sql/log_event.cc 2009-06-11 17:49:51 +0000 +++ b/sql/log_event.cc 2009-09-09 15:27:23 +0000 @@ -7427,6 +7427,13 @@ int Rows_log_event::do_apply_event(Relay const_cast<Relay_log_info*>(rli)->last_event_start_time= my_time(0); } + if (get_flags(STMT_END_F)) + { + error= rows_event_stmt_cleanup(rli, thd); + if (error) + DBUG_RETURN(error); + } + DBUG_RETURN(0); } @@ -7525,33 +7532,22 @@ Rows_log_event::do_update_pos(Relay_log_ if (get_flags(STMT_END_F)) { - if ((error= rows_event_stmt_cleanup(rli, thd)) == 0) - { - /* - Indicate that a statement is finished. - Step the group log position if we are not in a transaction, - otherwise increase the event log position. - */ - rli->stmt_done(log_pos, when); + /* + Indicate that a statement is finished. + Step the group log position if we are not in a transaction, + otherwise increase the event log position. + */ + rli->stmt_done(log_pos, when); - /* - Clear any errors pushed in thd->net.last_err* if for example "no key - found" (as this is allowed). This is a safety measure; apparently - those errors (e.g. when executing a Delete_rows_log_event of a - non-existing row, like in rpl_row_mystery22.test, - thd->net.last_error = "Can't find record in 't1'" and last_errno=1032) - do not become visible. We still prefer to wipe them out. - */ - thd->clear_error(); - } - else - { - rli->report(ERROR_LEVEL, error, - "Error in %s event: commit of row events failed, " - "table `%s`.`%s`", - get_type_str(), m_table->s->db.str, - m_table->s->table_name.str); - } + /* + Clear any errors pushed in thd->net.last_err* if for example "no key + found" (as this is allowed). This is a safety measure; apparently + those errors (e.g. when executing a Delete_rows_log_event of a + non-existing row, like in rpl_row_mystery22.test, + thd->net.last_error = "Can't find record in 't1'" and last_errno=1032) + do not become visible. We still prefer to wipe them out. + */ + thd->clear_error(); } else { === modified file 'sql/slave.cc' --- a/sql/slave.cc 2009-09-03 14:05:38 +0000 +++ b/sql/slave.cc 2009-09-09 15:27:23 +0000 @@ -1882,8 +1882,7 @@ static int has_temporary_error(THD *thd) @retval 2 No error calling ev->apply_event(), but error calling ev->update_pos(). */ -int apply_event_and_update_pos(Log_event* ev, THD* thd, Relay_log_info* rli, - bool skip) +int apply_event_and_update_pos(Log_event* ev, THD* thd, Relay_log_info* rli) { int exec_res= 0; @@ -1928,37 +1927,32 @@ int apply_event_and_update_pos(Log_event ev->when= my_time(0); ev->thd = thd; // because up to this point, ev->thd == 0 - if (skip) - { - int reason= ev->shall_skip(rli); - if (reason == Log_event::EVENT_SKIP_COUNT) - --rli->slave_skip_counter; - pthread_mutex_unlock(&rli->data_lock); - if (reason == Log_event::EVENT_SKIP_NOT) - exec_res= ev->apply_event(rli); + int reason= ev->shall_skip(rli); + if (reason == Log_event::EVENT_SKIP_COUNT) + --rli->slave_skip_counter; + pthread_mutex_unlock(&rli->data_lock); + if (reason == Log_event::EVENT_SKIP_NOT) + exec_res= ev->apply_event(rli); #ifndef DBUG_OFF - /* - This only prints information to the debug trace. + /* + This only prints information to the debug trace. - TODO: Print an informational message to the error log? - */ - static const char *const explain[] = { - // EVENT_SKIP_NOT, - "not skipped", - // EVENT_SKIP_IGNORE, - "skipped because event should be ignored", - // EVENT_SKIP_COUNT - "skipped because event skip counter was non-zero" - }; - DBUG_PRINT("info", ("OPTION_BEGIN: %d; IN_STMT: %d", - thd->options & OPTION_BEGIN ? 1 : 0, - rli->get_flag(Relay_log_info::IN_STMT))); - DBUG_PRINT("skip_event", ("%s event was %s", - ev->get_type_str(), explain[reason])); + TODO: Print an informational message to the error log? + */ + static const char *const explain[] = { + // EVENT_SKIP_NOT, + "not skipped", + // EVENT_SKIP_IGNORE, + "skipped because event should be ignored", + // EVENT_SKIP_COUNT + "skipped because event skip counter was non-zero" + }; + DBUG_PRINT("info", ("OPTION_BEGIN: %d; IN_STMT: %d", + thd->options & OPTION_BEGIN ? 1 : 0, + rli->get_flag(Relay_log_info::IN_STMT))); + DBUG_PRINT("skip_event", ("%s event was %s", + ev->get_type_str(), explain[reason])); #endif - } - else - exec_res= ev->apply_event(rli); DBUG_PRINT("info", ("apply_event error = %d", exec_res)); if (exec_res == 0) @@ -2078,7 +2072,7 @@ static int exec_relay_log_event(THD* thd delete ev; DBUG_RETURN(1); } - exec_res= apply_event_and_update_pos(ev, thd, rli, TRUE); + exec_res= apply_event_and_update_pos(ev, thd, rli); /* Format_description_log_event should not be deleted because it will be === modified file 'sql/slave.h' --- a/sql/slave.h 2008-12-10 09:02:25 +0000 +++ b/sql/slave.h 2009-09-09 15:27:23 +0000 @@ -190,8 +190,7 @@ int purge_relay_logs(Relay_log_info* rli void set_slave_thread_options(THD* thd); void set_slave_thread_default_charset(THD *thd, Relay_log_info const *rli); void rotate_relay_log(Master_info* mi); -int apply_event_and_update_pos(Log_event* ev, THD* thd, Relay_log_info* rli, - bool skip); +int apply_event_and_update_pos(Log_event* ev, THD* thd, Relay_log_info* rli); pthread_handler_t handle_slave_io(void *arg); pthread_handler_t handle_slave_sql(void *arg); === modified file 'sql/sql_binlog.cc' --- a/sql/sql_binlog.cc 2009-05-06 12:03:24 +0000 +++ b/sql/sql_binlog.cc 2009-09-09 15:27:23 +0000 @@ -56,17 +56,21 @@ void mysql_client_binlog_statement(THD* Format_description_event. */ my_bool have_fd_event= TRUE; - if (!thd->rli_fake) + int err; + Relay_log_info *rli; + + rli= thd->rli_fake; + if (!rli) { - thd->rli_fake= new Relay_log_info; + rli= thd->rli_fake= new Relay_log_info; #ifdef HAVE_valgrind - thd->rli_fake->is_fake= TRUE; + rli->is_fake= TRUE; #endif have_fd_event= FALSE; } - if (thd->rli_fake && !thd->rli_fake->relay_log.description_event_for_exec) + if (rli && !rli->relay_log.description_event_for_exec) { - thd->rli_fake->relay_log.description_event_for_exec= + rli->relay_log.description_event_for_exec= new Format_description_log_event(4); have_fd_event= FALSE; } @@ -78,16 +82,16 @@ void mysql_client_binlog_statement(THD* /* Out of memory check */ - if (!(thd->rli_fake && - thd->rli_fake->relay_log.description_event_for_exec && + if (!(rli && + rli->relay_log.description_event_for_exec && buf)) { my_error(ER_OUTOFMEMORY, MYF(0), 1); /* needed 1 bytes */ goto end; } - thd->rli_fake->sql_thd= thd; - thd->rli_fake->no_storage= TRUE; + rli->sql_thd= thd; + rli->no_storage= TRUE; for (char const *strptr= thd->lex->comment.str ; strptr < thd->lex->comment.str + thd->lex->comment.length ; ) @@ -170,8 +174,7 @@ void mysql_client_binlog_statement(THD* } ev= Log_event::read_log_event(bufptr, event_len, &error, - thd->rli_fake->relay_log. - description_event_for_exec); + rli->relay_log.description_event_for_exec); DBUG_PRINT("info",("binlog base64 err=%s", error)); if (!ev) @@ -209,9 +212,27 @@ void mysql_client_binlog_statement(THD* reporting. */ #if !defined(MYSQL_CLIENT) && defined(HAVE_REPLICATION) - if (apply_event_and_update_pos(ev, thd, thd->rli_fake, FALSE)) + err= ev->apply_event(rli); +#else + err= 0; +#endif + if (ev->get_type_code() == FORMAT_DESCRIPTION_EVENT) + { + /* + Install this as the format description event to use for any + following events. + */ + delete rli->relay_log.description_event_for_exec; + rli->relay_log.description_event_for_exec= + static_cast<Format_description_log_event *>(ev); + } + else { delete ev; + } + ev= 0; + if (err) + { /* TODO: Maybe a better error message since the BINLOG statement now contains several events. @@ -219,17 +240,6 @@ void mysql_client_binlog_statement(THD* my_error(ER_UNKNOWN_ERROR, MYF(0), "Error executing BINLOG statement"); goto end; } -#endif - - /* - Format_description_log_event should not be deleted because it - will be used to read info about the relay log's format; it - will be deleted when the SQL thread does not need it, - i.e. when this thread terminates. - */ - if (ev->get_type_code() != FORMAT_DESCRIPTION_EVENT) - delete ev; - ev= 0; } } @@ -238,7 +248,7 @@ void mysql_client_binlog_statement(THD* my_ok(thd); end: - thd->rli_fake->clear_tables_to_lock(); + rli->clear_tables_to_lock(); my_free(buf, MYF(MY_ALLOW_ZERO_PTR)); DBUG_VOID_RETURN; }

1 0

[Maria-developers] Updated (by Guest): mysqlbinlog: remove undesired effects of format description binlog statement (50)
by worklog-noreply＠askmonty.org 09 Sep '09

09 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: mysqlbinlog: remove undesired effects of format description binlog statement CREATION DATE..: Mon, 17 Aug 2009, 14:12 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 50 (http://askmonty.org/worklog/?tid=50) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 2 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Wed, 09 Sep 2009, 11:50)=-=- High Level Description modified. --- /tmp/wklog.50.old.11584 2009-09-09 11:50:12.000000000 +0300 +++ /tmp/wklog.50.new.11584 2009-09-09 11:50:12.000000000 +0300 @@ -5,7 +5,11 @@ This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. -2. When one applies "format description binlog statement" at the slave, it -will change the slave's server_id when applied. +2. When one executes a BINLOG statement (containing "format description binlog +statement" or other events), this will cause subsequent statements run in +the same session to be binlogged with the server id of the last event +executed, not with the real server id. This can cause problems like infinite +recursive application in a circular replication topology. This WL is to fix these issues. + -=-=(Knielsen - Tue, 08 Sep 2009, 15:07)=-=- Low Level Design modified. --- /tmp/wklog.50.old.19208 2009-09-08 15:07:09.000000000 +0300 +++ /tmp/wklog.50.new.19208 2009-09-08 15:07:09.000000000 +0300 @@ -1 +1,34 @@ +I think the fix for point 2 is to replace the call to +apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct +call to ev->apply_event(). However, I need to check two things to be sure this +is correct: + +1. The existing code does + + if (!ev->when) ev->when= my_time(0) + +Need to understand if this is needed/correct or not. + +2. The existing code does ev->update_pos(). I think this is redundant for +BINLOG statement (as it uses a fake rli structure), but I need to check to +make sure. + +Once this is done, point 1 may no longer be needed. The user can use +--base64-output=never when applying the output to a 5.0 server, and omit that +option when applying the output to a 5.1 server. There should be no need to +omit the format description event in the output when other BINLOG statements +for row events are present, as it no longer changes the server id. In fact the +format description event is required to be able to execute other BINLOG +statements, as its purpose is to define the binary format of the events +contained in these statements. + +Alternatively, we could implement that if the format description event of the +source binlog has server version < 5.1, and --base64-output=auto (default), +then the format description event is omitted (and should any BINLOG statement +be needed, unlikely as it is from a 5.0 server, we will need to throw an +error). + +The binlog format version for 5.0 and 5.1 is actually the same, hence the need +to look at server version to guess if the format description event can be +omitted. -=-=(Knielsen - Tue, 08 Sep 2009, 14:29)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.17531 2009-09-08 14:29:30.000000000 +0300 +++ /tmp/wklog.50.new.17531 2009-09-08 14:29:30.000000000 +0300 @@ -12,3 +12,37 @@ One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? +First, the problem is not the format description log event. In fact all log +events have their own value of server_id, and every event in the BINLOG +statement is executed with its own id as server_id. + +It seems it was introduced deliberately in the patch for Bug#32407. This patch +makes sql thread and BINLOG statement share code path for executing the binary +event, even though the bug is really about something different (outputting +format description event to allow proper execution of other BINLOG +statements). + +Nevertheless, I think using the server_id in the events of a BINLOG statement +is wrong: + +1. It is different behaviour from executing normal statement-based events not +written using BINLOG. + +2. It causes the possibility for infinite cycle of replication in a cyclic +replication setup, if the server ids in BINLOG are different from all other +server ids in the cycle. + +3. The functionality of applying events with original server id from the event +is still available by pointing the slave thread to the binlog. The +functionality of applying row-based binlog events with server id of the server +executing the BINLOG statements is not otherwise available. + +In fact most of the code from the slave thread that is now also run when +executing BINLOG statements is redundant, or does the wrong thing (like +updating binlog position). + +Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be +able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is +it to be able to simulate the effect of the slave sql thread? I think the +former is the most important one. + -=-=(Knielsen - Mon, 07 Sep 2009, 10:06)=-=- Research worklog, bugs, and existing server code. Worked 2 hours and estimate 0 hours remain (original estimate increased by 2 hours). -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- Dependency created: 39 now depends on 50 -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.11389 2009-08-17 14:13:05.000000000 +0300 +++ /tmp/wklog.50.new.11389 2009-08-17 14:13:05.000000000 +0300 @@ -1 +1,14 @@ +First item +---------- +AFAIU what needs to be done is: +1. record a source server version (it is in the first binlog event). +2. don't emit a BINLOG statement if the recorded version number is 5.0.x or + below. + +and we'll get a 5.0-applicable binlog. + +Second item +----------- +One question that one needs to sort out before disabling server_id change is +why it was put there in the first place? Should it be always removed? DESCRIPTION: According to complaints in BUG#46640: 1. mysqlbinlog will emit a 5.0-incompatible "format description binlog statement" even when reading a binary log that was produced by 5.0. This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. 2. When one executes a BINLOG statement (containing "format description binlog statement" or other events), this will cause subsequent statements run in the same session to be binlogged with the server id of the last event executed, not with the real server id. This can cause problems like infinite recursive application in a circular replication topology. This WL is to fix these issues. HIGH-LEVEL SPECIFICATION: First item ---------- AFAIU what needs to be done is: 1. record a source server version (it is in the first binlog event). 2. don't emit a BINLOG statement if the recorded version number is 5.0.x or below. and we'll get a 5.0-applicable binlog. Second item ----------- One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? First, the problem is not the format description log event. In fact all log events have their own value of server_id, and every event in the BINLOG statement is executed with its own id as server_id. It seems it was introduced deliberately in the patch for Bug#32407. This patch makes sql thread and BINLOG statement share code path for executing the binary event, even though the bug is really about something different (outputting format description event to allow proper execution of other BINLOG statements). Nevertheless, I think using the server_id in the events of a BINLOG statement is wrong: 1. It is different behaviour from executing normal statement-based events not written using BINLOG. 2. It causes the possibility for infinite cycle of replication in a cyclic replication setup, if the server ids in BINLOG are different from all other server ids in the cycle. 3. The functionality of applying events with original server id from the event is still available by pointing the slave thread to the binlog. The functionality of applying row-based binlog events with server id of the server executing the BINLOG statements is not otherwise available. In fact most of the code from the slave thread that is now also run when executing BINLOG statements is redundant, or does the wrong thing (like updating binlog position). Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is it to be able to simulate the effect of the slave sql thread? I think the former is the most important one. LOW-LEVEL DESIGN: I think the fix for point 2 is to replace the call to apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct call to ev->apply_event(). However, I need to check two things to be sure this is correct: 1. The existing code does if (!ev->when) ev->when= my_time(0) Need to understand if this is needed/correct or not. 2. The existing code does ev->update_pos(). I think this is redundant for BINLOG statement (as it uses a fake rli structure), but I need to check to make sure. Once this is done, point 1 may no longer be needed. The user can use --base64-output=never when applying the output to a 5.0 server, and omit that option when applying the output to a 5.1 server. There should be no need to omit the format description event in the output when other BINLOG statements for row events are present, as it no longer changes the server id. In fact the format description event is required to be able to execute other BINLOG statements, as its purpose is to define the binary format of the events contained in these statements. Alternatively, we could implement that if the format description event of the source binlog has server version < 5.1, and --base64-output=auto (default), then the format description event is omitted (and should any BINLOG statement be needed, unlikely as it is from a 5.0 server, we will need to throw an error). The binlog format version for 5.0 and 5.1 is actually the same, hence the need to look at server version to guess if the format description event can be omitted. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): mysqlbinlog: remove undesired effects of format description binlog statement (50)
by worklog-noreply＠askmonty.org 09 Sep '09

09 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: mysqlbinlog: remove undesired effects of format description binlog statement CREATION DATE..: Mon, 17 Aug 2009, 14:12 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 50 (http://askmonty.org/worklog/?tid=50) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 2 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Wed, 09 Sep 2009, 11:50)=-=- High Level Description modified. --- /tmp/wklog.50.old.11584 2009-09-09 11:50:12.000000000 +0300 +++ /tmp/wklog.50.new.11584 2009-09-09 11:50:12.000000000 +0300 @@ -5,7 +5,11 @@ This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. -2. When one applies "format description binlog statement" at the slave, it -will change the slave's server_id when applied. +2. When one executes a BINLOG statement (containing "format description binlog +statement" or other events), this will cause subsequent statements run in +the same session to be binlogged with the server id of the last event +executed, not with the real server id. This can cause problems like infinite +recursive application in a circular replication topology. This WL is to fix these issues. + -=-=(Knielsen - Tue, 08 Sep 2009, 15:07)=-=- Low Level Design modified. --- /tmp/wklog.50.old.19208 2009-09-08 15:07:09.000000000 +0300 +++ /tmp/wklog.50.new.19208 2009-09-08 15:07:09.000000000 +0300 @@ -1 +1,34 @@ +I think the fix for point 2 is to replace the call to +apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct +call to ev->apply_event(). However, I need to check two things to be sure this +is correct: + +1. The existing code does + + if (!ev->when) ev->when= my_time(0) + +Need to understand if this is needed/correct or not. + +2. The existing code does ev->update_pos(). I think this is redundant for +BINLOG statement (as it uses a fake rli structure), but I need to check to +make sure. + +Once this is done, point 1 may no longer be needed. The user can use +--base64-output=never when applying the output to a 5.0 server, and omit that +option when applying the output to a 5.1 server. There should be no need to +omit the format description event in the output when other BINLOG statements +for row events are present, as it no longer changes the server id. In fact the +format description event is required to be able to execute other BINLOG +statements, as its purpose is to define the binary format of the events +contained in these statements. + +Alternatively, we could implement that if the format description event of the +source binlog has server version < 5.1, and --base64-output=auto (default), +then the format description event is omitted (and should any BINLOG statement +be needed, unlikely as it is from a 5.0 server, we will need to throw an +error). + +The binlog format version for 5.0 and 5.1 is actually the same, hence the need +to look at server version to guess if the format description event can be +omitted. -=-=(Knielsen - Tue, 08 Sep 2009, 14:29)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.17531 2009-09-08 14:29:30.000000000 +0300 +++ /tmp/wklog.50.new.17531 2009-09-08 14:29:30.000000000 +0300 @@ -12,3 +12,37 @@ One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? +First, the problem is not the format description log event. In fact all log +events have their own value of server_id, and every event in the BINLOG +statement is executed with its own id as server_id. + +It seems it was introduced deliberately in the patch for Bug#32407. This patch +makes sql thread and BINLOG statement share code path for executing the binary +event, even though the bug is really about something different (outputting +format description event to allow proper execution of other BINLOG +statements). + +Nevertheless, I think using the server_id in the events of a BINLOG statement +is wrong: + +1. It is different behaviour from executing normal statement-based events not +written using BINLOG. + +2. It causes the possibility for infinite cycle of replication in a cyclic +replication setup, if the server ids in BINLOG are different from all other +server ids in the cycle. + +3. The functionality of applying events with original server id from the event +is still available by pointing the slave thread to the binlog. The +functionality of applying row-based binlog events with server id of the server +executing the BINLOG statements is not otherwise available. + +In fact most of the code from the slave thread that is now also run when +executing BINLOG statements is redundant, or does the wrong thing (like +updating binlog position). + +Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be +able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is +it to be able to simulate the effect of the slave sql thread? I think the +former is the most important one. + -=-=(Knielsen - Mon, 07 Sep 2009, 10:06)=-=- Research worklog, bugs, and existing server code. Worked 2 hours and estimate 0 hours remain (original estimate increased by 2 hours). -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- Dependency created: 39 now depends on 50 -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.11389 2009-08-17 14:13:05.000000000 +0300 +++ /tmp/wklog.50.new.11389 2009-08-17 14:13:05.000000000 +0300 @@ -1 +1,14 @@ +First item +---------- +AFAIU what needs to be done is: +1. record a source server version (it is in the first binlog event). +2. don't emit a BINLOG statement if the recorded version number is 5.0.x or + below. + +and we'll get a 5.0-applicable binlog. + +Second item +----------- +One question that one needs to sort out before disabling server_id change is +why it was put there in the first place? Should it be always removed? DESCRIPTION: According to complaints in BUG#46640: 1. mysqlbinlog will emit a 5.0-incompatible "format description binlog statement" even when reading a binary log that was produced by 5.0. This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. 2. When one executes a BINLOG statement (containing "format description binlog statement" or other events), this will cause subsequent statements run in the same session to be binlogged with the server id of the last event executed, not with the real server id. This can cause problems like infinite recursive application in a circular replication topology. This WL is to fix these issues. HIGH-LEVEL SPECIFICATION: First item ---------- AFAIU what needs to be done is: 1. record a source server version (it is in the first binlog event). 2. don't emit a BINLOG statement if the recorded version number is 5.0.x or below. and we'll get a 5.0-applicable binlog. Second item ----------- One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? First, the problem is not the format description log event. In fact all log events have their own value of server_id, and every event in the BINLOG statement is executed with its own id as server_id. It seems it was introduced deliberately in the patch for Bug#32407. This patch makes sql thread and BINLOG statement share code path for executing the binary event, even though the bug is really about something different (outputting format description event to allow proper execution of other BINLOG statements). Nevertheless, I think using the server_id in the events of a BINLOG statement is wrong: 1. It is different behaviour from executing normal statement-based events not written using BINLOG. 2. It causes the possibility for infinite cycle of replication in a cyclic replication setup, if the server ids in BINLOG are different from all other server ids in the cycle. 3. The functionality of applying events with original server id from the event is still available by pointing the slave thread to the binlog. The functionality of applying row-based binlog events with server id of the server executing the BINLOG statements is not otherwise available. In fact most of the code from the slave thread that is now also run when executing BINLOG statements is redundant, or does the wrong thing (like updating binlog position). Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is it to be able to simulate the effect of the slave sql thread? I think the former is the most important one. LOW-LEVEL DESIGN: I think the fix for point 2 is to replace the call to apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct call to ev->apply_event(). However, I need to check two things to be sure this is correct: 1. The existing code does if (!ev->when) ev->when= my_time(0) Need to understand if this is needed/correct or not. 2. The existing code does ev->update_pos(). I think this is redundant for BINLOG statement (as it uses a fake rli structure), but I need to check to make sure. Once this is done, point 1 may no longer be needed. The user can use --base64-output=never when applying the output to a 5.0 server, and omit that option when applying the output to a 5.1 server. There should be no need to omit the format description event in the output when other BINLOG statements for row events are present, as it no longer changes the server id. In fact the format description event is required to be able to execute other BINLOG statements, as its purpose is to define the binary format of the events contained in these statements. Alternatively, we could implement that if the format description event of the source binlog has server version < 5.1, and --base64-output=auto (default), then the format description event is omitted (and should any BINLOG statement be needed, unlikely as it is from a 5.0 server, we will need to throw an error). The binlog format version for 5.0 and 5.1 is actually the same, hence the need to look at server version to guess if the format description event can be omitted. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

Re: [Maria-developers] Rev 2721: Merge xtradb-7 -> MariaDB in file:///home/psergey/bzr-new/mysql-5.1-maria-contd2/
by Sergey Petrunya 08 Sep '09

08 Sep '09

Hi Peter, On Tue, Sep 08, 2009 at 06:33:00PM +0200, Peter Laursen wrote: > @Sergey > . > does this mean that we can now check out code, compile and XtraDB will be > functional? > (cc: to Khusboo as she is assigned as build-engineer for the Windows MariaDB > planned to be provided by Webyog) No, not yet: I haven't pushed this yet, because the result of the merge has certain issues. There will be more commits and then push. Besides the merge, we have also two known windows problems, as mentioned here: http://askmonty.org/wiki/index.php/MariaDB_5.1_TODO#Bugfixes We're working on these also. Do you want me to notify you when lp:maria is ready for windows packaging? BR Sergey -- Sergey Petrunia, Software Developer Monty Program AB, http://askmonty.org Blog: http://s.petrunia.net/blog

2 2

[Maria-developers] Rev 2722: Post-merge fix in file:///home/psergey/bzr-new/mysql-5.1-maria-contd2/
by Sergey Petrunya 08 Sep '09

08 Sep '09

At file:///home/psergey/bzr-new/mysql-5.1-maria-contd2/ ------------------------------------------------------------ revno: 2722 revision-id: psergey(a)askmonty.org-20090908162138-wtd6r54dkcp4hg5a parent: psergey(a)askmonty.org-20090908160458-fglojzacu8inqb9q committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: mysql-5.1-maria-contd2 timestamp: Tue 2009-09-08 20:21:38 +0400 message: Post-merge fix === removed directory 'mysql-test.percona'

1 0

[Maria-developers] Updated (by Knielsen): mysqlbinlog: remove undesired effects of format description binlog statement (50)
by worklog-noreply＠askmonty.org 08 Sep '09

08 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: mysqlbinlog: remove undesired effects of format description binlog statement CREATION DATE..: Mon, 17 Aug 2009, 14:12 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 50 (http://askmonty.org/worklog/?tid=50) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 2 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Tue, 08 Sep 2009, 15:07)=-=- Low Level Design modified. --- /tmp/wklog.50.old.19208 2009-09-08 15:07:09.000000000 +0300 +++ /tmp/wklog.50.new.19208 2009-09-08 15:07:09.000000000 +0300 @@ -1 +1,34 @@ +I think the fix for point 2 is to replace the call to +apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct +call to ev->apply_event(). However, I need to check two things to be sure this +is correct: + +1. The existing code does + + if (!ev->when) ev->when= my_time(0) + +Need to understand if this is needed/correct or not. + +2. The existing code does ev->update_pos(). I think this is redundant for +BINLOG statement (as it uses a fake rli structure), but I need to check to +make sure. + +Once this is done, point 1 may no longer be needed. The user can use +--base64-output=never when applying the output to a 5.0 server, and omit that +option when applying the output to a 5.1 server. There should be no need to +omit the format description event in the output when other BINLOG statements +for row events are present, as it no longer changes the server id. In fact the +format description event is required to be able to execute other BINLOG +statements, as its purpose is to define the binary format of the events +contained in these statements. + +Alternatively, we could implement that if the format description event of the +source binlog has server version < 5.1, and --base64-output=auto (default), +then the format description event is omitted (and should any BINLOG statement +be needed, unlikely as it is from a 5.0 server, we will need to throw an +error). + +The binlog format version for 5.0 and 5.1 is actually the same, hence the need +to look at server version to guess if the format description event can be +omitted. -=-=(Knielsen - Tue, 08 Sep 2009, 14:29)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.17531 2009-09-08 14:29:30.000000000 +0300 +++ /tmp/wklog.50.new.17531 2009-09-08 14:29:30.000000000 +0300 @@ -12,3 +12,37 @@ One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? +First, the problem is not the format description log event. In fact all log +events have their own value of server_id, and every event in the BINLOG +statement is executed with its own id as server_id. + +It seems it was introduced deliberately in the patch for Bug#32407. This patch +makes sql thread and BINLOG statement share code path for executing the binary +event, even though the bug is really about something different (outputting +format description event to allow proper execution of other BINLOG +statements). + +Nevertheless, I think using the server_id in the events of a BINLOG statement +is wrong: + +1. It is different behaviour from executing normal statement-based events not +written using BINLOG. + +2. It causes the possibility for infinite cycle of replication in a cyclic +replication setup, if the server ids in BINLOG are different from all other +server ids in the cycle. + +3. The functionality of applying events with original server id from the event +is still available by pointing the slave thread to the binlog. The +functionality of applying row-based binlog events with server id of the server +executing the BINLOG statements is not otherwise available. + +In fact most of the code from the slave thread that is now also run when +executing BINLOG statements is redundant, or does the wrong thing (like +updating binlog position). + +Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be +able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is +it to be able to simulate the effect of the slave sql thread? I think the +former is the most important one. + -=-=(Knielsen - Mon, 07 Sep 2009, 10:06)=-=- Research worklog, bugs, and existing server code. Worked 2 hours and estimate 0 hours remain (original estimate increased by 2 hours). -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- Dependency created: 39 now depends on 50 -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.11389 2009-08-17 14:13:05.000000000 +0300 +++ /tmp/wklog.50.new.11389 2009-08-17 14:13:05.000000000 +0300 @@ -1 +1,14 @@ +First item +---------- +AFAIU what needs to be done is: +1. record a source server version (it is in the first binlog event). +2. don't emit a BINLOG statement if the recorded version number is 5.0.x or + below. + +and we'll get a 5.0-applicable binlog. + +Second item +----------- +One question that one needs to sort out before disabling server_id change is +why it was put there in the first place? Should it be always removed? DESCRIPTION: According to complaints in BUG#46640: 1. mysqlbinlog will emit a 5.0-incompatible "format description binlog statement" even when reading a binary log that was produced by 5.0. This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. 2. When one applies "format description binlog statement" at the slave, it will change the slave's server_id when applied. This WL is to fix these issues. HIGH-LEVEL SPECIFICATION: First item ---------- AFAIU what needs to be done is: 1. record a source server version (it is in the first binlog event). 2. don't emit a BINLOG statement if the recorded version number is 5.0.x or below. and we'll get a 5.0-applicable binlog. Second item ----------- One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? First, the problem is not the format description log event. In fact all log events have their own value of server_id, and every event in the BINLOG statement is executed with its own id as server_id. It seems it was introduced deliberately in the patch for Bug#32407. This patch makes sql thread and BINLOG statement share code path for executing the binary event, even though the bug is really about something different (outputting format description event to allow proper execution of other BINLOG statements). Nevertheless, I think using the server_id in the events of a BINLOG statement is wrong: 1. It is different behaviour from executing normal statement-based events not written using BINLOG. 2. It causes the possibility for infinite cycle of replication in a cyclic replication setup, if the server ids in BINLOG are different from all other server ids in the cycle. 3. The functionality of applying events with original server id from the event is still available by pointing the slave thread to the binlog. The functionality of applying row-based binlog events with server id of the server executing the BINLOG statements is not otherwise available. In fact most of the code from the slave thread that is now also run when executing BINLOG statements is redundant, or does the wrong thing (like updating binlog position). Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is it to be able to simulate the effect of the slave sql thread? I think the former is the most important one. LOW-LEVEL DESIGN: I think the fix for point 2 is to replace the call to apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct call to ev->apply_event(). However, I need to check two things to be sure this is correct: 1. The existing code does if (!ev->when) ev->when= my_time(0) Need to understand if this is needed/correct or not. 2. The existing code does ev->update_pos(). I think this is redundant for BINLOG statement (as it uses a fake rli structure), but I need to check to make sure. Once this is done, point 1 may no longer be needed. The user can use --base64-output=never when applying the output to a 5.0 server, and omit that option when applying the output to a 5.1 server. There should be no need to omit the format description event in the output when other BINLOG statements for row events are present, as it no longer changes the server id. In fact the format description event is required to be able to execute other BINLOG statements, as its purpose is to define the binary format of the events contained in these statements. Alternatively, we could implement that if the format description event of the source binlog has server version < 5.1, and --base64-output=auto (default), then the format description event is omitted (and should any BINLOG statement be needed, unlikely as it is from a 5.0 server, we will need to throw an error). The binlog format version for 5.0 and 5.1 is actually the same, hence the need to look at server version to guess if the format description event can be omitted. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Knielsen): mysqlbinlog: remove undesired effects of format description binlog statement (50)
by worklog-noreply＠askmonty.org 08 Sep '09

08 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: mysqlbinlog: remove undesired effects of format description binlog statement CREATION DATE..: Mon, 17 Aug 2009, 14:12 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 50 (http://askmonty.org/worklog/?tid=50) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 2 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Tue, 08 Sep 2009, 15:07)=-=- Low Level Design modified. --- /tmp/wklog.50.old.19208 2009-09-08 15:07:09.000000000 +0300 +++ /tmp/wklog.50.new.19208 2009-09-08 15:07:09.000000000 +0300 @@ -1 +1,34 @@ +I think the fix for point 2 is to replace the call to +apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct +call to ev->apply_event(). However, I need to check two things to be sure this +is correct: + +1. The existing code does + + if (!ev->when) ev->when= my_time(0) + +Need to understand if this is needed/correct or not. + +2. The existing code does ev->update_pos(). I think this is redundant for +BINLOG statement (as it uses a fake rli structure), but I need to check to +make sure. + +Once this is done, point 1 may no longer be needed. The user can use +--base64-output=never when applying the output to a 5.0 server, and omit that +option when applying the output to a 5.1 server. There should be no need to +omit the format description event in the output when other BINLOG statements +for row events are present, as it no longer changes the server id. In fact the +format description event is required to be able to execute other BINLOG +statements, as its purpose is to define the binary format of the events +contained in these statements. + +Alternatively, we could implement that if the format description event of the +source binlog has server version < 5.1, and --base64-output=auto (default), +then the format description event is omitted (and should any BINLOG statement +be needed, unlikely as it is from a 5.0 server, we will need to throw an +error). + +The binlog format version for 5.0 and 5.1 is actually the same, hence the need +to look at server version to guess if the format description event can be +omitted. -=-=(Knielsen - Tue, 08 Sep 2009, 14:29)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.17531 2009-09-08 14:29:30.000000000 +0300 +++ /tmp/wklog.50.new.17531 2009-09-08 14:29:30.000000000 +0300 @@ -12,3 +12,37 @@ One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? +First, the problem is not the format description log event. In fact all log +events have their own value of server_id, and every event in the BINLOG +statement is executed with its own id as server_id. + +It seems it was introduced deliberately in the patch for Bug#32407. This patch +makes sql thread and BINLOG statement share code path for executing the binary +event, even though the bug is really about something different (outputting +format description event to allow proper execution of other BINLOG +statements). + +Nevertheless, I think using the server_id in the events of a BINLOG statement +is wrong: + +1. It is different behaviour from executing normal statement-based events not +written using BINLOG. + +2. It causes the possibility for infinite cycle of replication in a cyclic +replication setup, if the server ids in BINLOG are different from all other +server ids in the cycle. + +3. The functionality of applying events with original server id from the event +is still available by pointing the slave thread to the binlog. The +functionality of applying row-based binlog events with server id of the server +executing the BINLOG statements is not otherwise available. + +In fact most of the code from the slave thread that is now also run when +executing BINLOG statements is redundant, or does the wrong thing (like +updating binlog position). + +Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be +able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is +it to be able to simulate the effect of the slave sql thread? I think the +former is the most important one. + -=-=(Knielsen - Mon, 07 Sep 2009, 10:06)=-=- Research worklog, bugs, and existing server code. Worked 2 hours and estimate 0 hours remain (original estimate increased by 2 hours). -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- Dependency created: 39 now depends on 50 -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.11389 2009-08-17 14:13:05.000000000 +0300 +++ /tmp/wklog.50.new.11389 2009-08-17 14:13:05.000000000 +0300 @@ -1 +1,14 @@ +First item +---------- +AFAIU what needs to be done is: +1. record a source server version (it is in the first binlog event). +2. don't emit a BINLOG statement if the recorded version number is 5.0.x or + below. + +and we'll get a 5.0-applicable binlog. + +Second item +----------- +One question that one needs to sort out before disabling server_id change is +why it was put there in the first place? Should it be always removed? DESCRIPTION: According to complaints in BUG#46640: 1. mysqlbinlog will emit a 5.0-incompatible "format description binlog statement" even when reading a binary log that was produced by 5.0. This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. 2. When one applies "format description binlog statement" at the slave, it will change the slave's server_id when applied. This WL is to fix these issues. HIGH-LEVEL SPECIFICATION: First item ---------- AFAIU what needs to be done is: 1. record a source server version (it is in the first binlog event). 2. don't emit a BINLOG statement if the recorded version number is 5.0.x or below. and we'll get a 5.0-applicable binlog. Second item ----------- One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? First, the problem is not the format description log event. In fact all log events have their own value of server_id, and every event in the BINLOG statement is executed with its own id as server_id. It seems it was introduced deliberately in the patch for Bug#32407. This patch makes sql thread and BINLOG statement share code path for executing the binary event, even though the bug is really about something different (outputting format description event to allow proper execution of other BINLOG statements). Nevertheless, I think using the server_id in the events of a BINLOG statement is wrong: 1. It is different behaviour from executing normal statement-based events not written using BINLOG. 2. It causes the possibility for infinite cycle of replication in a cyclic replication setup, if the server ids in BINLOG are different from all other server ids in the cycle. 3. The functionality of applying events with original server id from the event is still available by pointing the slave thread to the binlog. The functionality of applying row-based binlog events with server id of the server executing the BINLOG statements is not otherwise available. In fact most of the code from the slave thread that is now also run when executing BINLOG statements is redundant, or does the wrong thing (like updating binlog position). Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is it to be able to simulate the effect of the slave sql thread? I think the former is the most important one. LOW-LEVEL DESIGN: I think the fix for point 2 is to replace the call to apply_event_and_update_pos() in mysql_client_binlog_statement() with a direct call to ev->apply_event(). However, I need to check two things to be sure this is correct: 1. The existing code does if (!ev->when) ev->when= my_time(0) Need to understand if this is needed/correct or not. 2. The existing code does ev->update_pos(). I think this is redundant for BINLOG statement (as it uses a fake rli structure), but I need to check to make sure. Once this is done, point 1 may no longer be needed. The user can use --base64-output=never when applying the output to a 5.0 server, and omit that option when applying the output to a 5.1 server. There should be no need to omit the format description event in the output when other BINLOG statements for row events are present, as it no longer changes the server id. In fact the format description event is required to be able to execute other BINLOG statements, as its purpose is to define the binary format of the events contained in these statements. Alternatively, we could implement that if the format description event of the source binlog has server version < 5.1, and --base64-output=auto (default), then the format description event is omitted (and should any BINLOG statement be needed, unlikely as it is from a 5.0 server, we will need to throw an error). The binlog format version for 5.0 and 5.1 is actually the same, hence the need to look at server version to guess if the format description event can be omitted. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Knielsen): mysqlbinlog: remove undesired effects of format description binlog statement (50)
by worklog-noreply＠askmonty.org 08 Sep '09

08 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: mysqlbinlog: remove undesired effects of format description binlog statement CREATION DATE..: Mon, 17 Aug 2009, 14:12 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 50 (http://askmonty.org/worklog/?tid=50) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 2 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Tue, 08 Sep 2009, 14:29)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.17531 2009-09-08 14:29:30.000000000 +0300 +++ /tmp/wklog.50.new.17531 2009-09-08 14:29:30.000000000 +0300 @@ -12,3 +12,37 @@ One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? +First, the problem is not the format description log event. In fact all log +events have their own value of server_id, and every event in the BINLOG +statement is executed with its own id as server_id. + +It seems it was introduced deliberately in the patch for Bug#32407. This patch +makes sql thread and BINLOG statement share code path for executing the binary +event, even though the bug is really about something different (outputting +format description event to allow proper execution of other BINLOG +statements). + +Nevertheless, I think using the server_id in the events of a BINLOG statement +is wrong: + +1. It is different behaviour from executing normal statement-based events not +written using BINLOG. + +2. It causes the possibility for infinite cycle of replication in a cyclic +replication setup, if the server ids in BINLOG are different from all other +server ids in the cycle. + +3. The functionality of applying events with original server id from the event +is still available by pointing the slave thread to the binlog. The +functionality of applying row-based binlog events with server id of the server +executing the BINLOG statements is not otherwise available. + +In fact most of the code from the slave thread that is now also run when +executing BINLOG statements is redundant, or does the wrong thing (like +updating binlog position). + +Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be +able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is +it to be able to simulate the effect of the slave sql thread? I think the +former is the most important one. + -=-=(Knielsen - Mon, 07 Sep 2009, 10:06)=-=- Research worklog, bugs, and existing server code. Worked 2 hours and estimate 0 hours remain (original estimate increased by 2 hours). -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- Dependency created: 39 now depends on 50 -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.11389 2009-08-17 14:13:05.000000000 +0300 +++ /tmp/wklog.50.new.11389 2009-08-17 14:13:05.000000000 +0300 @@ -1 +1,14 @@ +First item +---------- +AFAIU what needs to be done is: +1. record a source server version (it is in the first binlog event). +2. don't emit a BINLOG statement if the recorded version number is 5.0.x or + below. + +and we'll get a 5.0-applicable binlog. + +Second item +----------- +One question that one needs to sort out before disabling server_id change is +why it was put there in the first place? Should it be always removed? DESCRIPTION: According to complaints in BUG#46640: 1. mysqlbinlog will emit a 5.0-incompatible "format description binlog statement" even when reading a binary log that was produced by 5.0. This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. 2. When one applies "format description binlog statement" at the slave, it will change the slave's server_id when applied. This WL is to fix these issues. HIGH-LEVEL SPECIFICATION: First item ---------- AFAIU what needs to be done is: 1. record a source server version (it is in the first binlog event). 2. don't emit a BINLOG statement if the recorded version number is 5.0.x or below. and we'll get a 5.0-applicable binlog. Second item ----------- One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? First, the problem is not the format description log event. In fact all log events have their own value of server_id, and every event in the BINLOG statement is executed with its own id as server_id. It seems it was introduced deliberately in the patch for Bug#32407. This patch makes sql thread and BINLOG statement share code path for executing the binary event, even though the bug is really about something different (outputting format description event to allow proper execution of other BINLOG statements). Nevertheless, I think using the server_id in the events of a BINLOG statement is wrong: 1. It is different behaviour from executing normal statement-based events not written using BINLOG. 2. It causes the possibility for infinite cycle of replication in a cyclic replication setup, if the server ids in BINLOG are different from all other server ids in the cycle. 3. The functionality of applying events with original server id from the event is still available by pointing the slave thread to the binlog. The functionality of applying row-based binlog events with server id of the server executing the BINLOG statements is not otherwise available. In fact most of the code from the slave thread that is now also run when executing BINLOG statements is redundant, or does the wrong thing (like updating binlog position). Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is it to be able to simulate the effect of the slave sql thread? I think the former is the most important one. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Knielsen): mysqlbinlog: remove undesired effects of format description binlog statement (50)
by worklog-noreply＠askmonty.org 08 Sep '09

08 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: mysqlbinlog: remove undesired effects of format description binlog statement CREATION DATE..: Mon, 17 Aug 2009, 14:12 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 50 (http://askmonty.org/worklog/?tid=50) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 2 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Tue, 08 Sep 2009, 14:29)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.17531 2009-09-08 14:29:30.000000000 +0300 +++ /tmp/wklog.50.new.17531 2009-09-08 14:29:30.000000000 +0300 @@ -12,3 +12,37 @@ One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? +First, the problem is not the format description log event. In fact all log +events have their own value of server_id, and every event in the BINLOG +statement is executed with its own id as server_id. + +It seems it was introduced deliberately in the patch for Bug#32407. This patch +makes sql thread and BINLOG statement share code path for executing the binary +event, even though the bug is really about something different (outputting +format description event to allow proper execution of other BINLOG +statements). + +Nevertheless, I think using the server_id in the events of a BINLOG statement +is wrong: + +1. It is different behaviour from executing normal statement-based events not +written using BINLOG. + +2. It causes the possibility for infinite cycle of replication in a cyclic +replication setup, if the server ids in BINLOG are different from all other +server ids in the cycle. + +3. The functionality of applying events with original server id from the event +is still available by pointing the slave thread to the binlog. The +functionality of applying row-based binlog events with server id of the server +executing the BINLOG statements is not otherwise available. + +In fact most of the code from the slave thread that is now also run when +executing BINLOG statements is redundant, or does the wrong thing (like +updating binlog position). + +Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be +able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is +it to be able to simulate the effect of the slave sql thread? I think the +former is the most important one. + -=-=(Knielsen - Mon, 07 Sep 2009, 10:06)=-=- Research worklog, bugs, and existing server code. Worked 2 hours and estimate 0 hours remain (original estimate increased by 2 hours). -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- Dependency created: 39 now depends on 50 -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.11389 2009-08-17 14:13:05.000000000 +0300 +++ /tmp/wklog.50.new.11389 2009-08-17 14:13:05.000000000 +0300 @@ -1 +1,14 @@ +First item +---------- +AFAIU what needs to be done is: +1. record a source server version (it is in the first binlog event). +2. don't emit a BINLOG statement if the recorded version number is 5.0.x or + below. + +and we'll get a 5.0-applicable binlog. + +Second item +----------- +One question that one needs to sort out before disabling server_id change is +why it was put there in the first place? Should it be always removed? DESCRIPTION: According to complaints in BUG#46640: 1. mysqlbinlog will emit a 5.0-incompatible "format description binlog statement" even when reading a binary log that was produced by 5.0. This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. 2. When one applies "format description binlog statement" at the slave, it will change the slave's server_id when applied. This WL is to fix these issues. HIGH-LEVEL SPECIFICATION: First item ---------- AFAIU what needs to be done is: 1. record a source server version (it is in the first binlog event). 2. don't emit a BINLOG statement if the recorded version number is 5.0.x or below. and we'll get a 5.0-applicable binlog. Second item ----------- One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? First, the problem is not the format description log event. In fact all log events have their own value of server_id, and every event in the BINLOG statement is executed with its own id as server_id. It seems it was introduced deliberately in the patch for Bug#32407. This patch makes sql thread and BINLOG statement share code path for executing the binary event, even though the bug is really about something different (outputting format description event to allow proper execution of other BINLOG statements). Nevertheless, I think using the server_id in the events of a BINLOG statement is wrong: 1. It is different behaviour from executing normal statement-based events not written using BINLOG. 2. It causes the possibility for infinite cycle of replication in a cyclic replication setup, if the server ids in BINLOG are different from all other server ids in the cycle. 3. The functionality of applying events with original server id from the event is still available by pointing the slave thread to the binlog. The functionality of applying row-based binlog events with server id of the server executing the BINLOG statements is not otherwise available. In fact most of the code from the slave thread that is now also run when executing BINLOG statements is redundant, or does the wrong thing (like updating binlog position). Maybe there is a confusion on the purpose of the BINLOG statement. Is it to be able to apply row-based events manually to a server (mysqlbinlog|mysql)? Or is it to be able to simulate the effect of the slave sql thread? I think the former is the most important one. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

Re: [Maria-developers] MariaDB release help
by Kristian Nielsen 08 Sep '09

08 Sep '09

Hakan Kuecuekyilmaz <hakan(a)askmonty.org> writes: > Where do you need some help for our release? Looking at http://askmonty.org/wiki/index.php/MariaDB_5.1_TODO, seems it is currently mostly the mysql-5.1.38 merge. One thing that would be useful is to try some additional testing on top of what is already in Buildbot, as there is quite a lot missing... some of this might be better done after the merge is pushed though, but you could start preparing. Maybe you could check what additional test suites/tools are already available that could be used? Stuff like unit tests, mysql-test-run --ps-protocol, additional test suites... One thing in particular is to prepare to test the actual release. Releases are built in a two-step process, first build a source tarball, and then build the binaries from those sources. Buildbot builds directly from a bzr checkout. And Binaries are run from the installed location, while Buildbot runs the tests from the build directory. It would be really embarrassing if some major part of our binaries were released completely broken due to some file or something not being included and no testing at all done :-(. So we could use some tools to test stuff like this. Eventually we want this in Buildbot of course. Concrete suggestion: 1. Testing on installed mysqld. Run `BUILD/compile-dist && make dist`. Unpack those sources and run `./configure <options> && make && make install`. Run `make clean` (to catch bad references into build dir). Then run the testsuite on the installed mysqld. 2. What we used to call "smoke test" at MySQL: Build a binary package, then install it on some machine and see that it can start and stop. Basically testing the package stuff (directory structure, mysqld_safe, ...) which is not covered by the test suite. See http://askmonty.org/wiki/index.php/MariaDB::PackageBuild for how packages are built currently. > My idea to work on and look into it: Currently we have around 10 test > failures on Mac/PPC. If you want, I can start looking into those > failures. Where should we report failing tests? I would like to open > a bug for each test failure. Currently we seem to be using Launchpad for bugs. However, it does not sound all that useful to blindly report one bug per failure. Probably some of them are related with same root cause. If you can identify those root causes, then one bug each makes sense (maybe this is what you meant). Otherwise a single bug probably, just noting that someone needs to look into it when time is available. - Kristian.

2 1

[Maria-developers] [Branch ~maria-captains/maria/5.1] Rev 2730: Fix that thd->query_plan_flags was not initialised prior to executing non-query commands
by noreply＠launchpad.net 07 Sep '09

07 Sep '09

------------------------------------------------------------ revno: 2730 committer: knielsen(a)knielsen-hq.org branch nick: work timestamp: Mon 2009-09-07 15:13:58 +0200 message: Fix that thd->query_plan_flags was not initialised prior to executing non-query commands (also fixes a Valgrind warning in Buildbot). modified: sql/sql_parse.cc -- lp:maria https://code.launchpad.net/~maria-captains/maria/5.1 Your team Maria developers is subscribed to branch lp:maria. To unsubscribe from this branch go to https://code.launchpad.net/~maria-captains/maria/5.1/+edit-subscription.

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2730)
by knielsen＠knielsen-hq.org 07 Sep '09

07 Sep '09

#At lp:maria 2730 knielsen(a)knielsen-hq.org 2009-09-07 Fix that thd->query_plan_flags was not initialised prior to executing non-query commands (also fixes a Valgrind warning in Buildbot). modified: sql/sql_parse.cc === modified file 'sql/sql_parse.cc' --- a/sql/sql_parse.cc 2009-09-03 14:05:38 +0000 +++ b/sql/sql_parse.cc 2009-09-07 13:13:58 +0000 @@ -975,6 +975,7 @@ bool dispatch_command(enum enum_server_c the slow log only if opt_log_slow_admin_statements is set. */ thd->enable_slow_log= TRUE; + thd->query_plan_flags= QPLAN_INIT; thd->lex->sql_command= SQLCOM_END; /* to avoid confusing VIEW detectors */ thd->set_time(); VOID(pthread_mutex_lock(&LOCK_thread_count));

1 0

Re: [Maria-developers] [Branch ~maria-captains/maria/5.1] Rev 2726: Updated documentation files to reflect MariaDB and not the Maria storage engine or MySQL
by Kristian Nielsen 07 Sep '09

07 Sep '09

noreply(a)launchpad.net writes: > Merge authors: > Michael Widenius (monty) > ------------------------------------------------------------ > revno: 2726 [merge] > committer: Michael Widenius <monty(a)mysql.com> > branch nick: maria > timestamp: Thu 2009-09-03 17:56:46 +0300 > message: > Updated documentation files to reflect MariaDB and not the Maria storage engine or MySQL > Added (rewritten) patch from Percona to get extended statistics in slow.log: > === modified file 'sql/log.cc' > --- sql/log.cc 2009-04-25 10:05:32 +0000 > +++ sql/log.cc 2009-09-03 14:05:38 +0000 > @@ -987,6 +987,17 @@ > query_length= command_name[thd->command].length; > } > > + if (!query_length) > + { > + /* > + Not a real query; Reset counts for slow query logging > + (QQ: Wonder if this is really needed) > + */ > + thd->sent_row_count= thd->examined_row_count= 0; > + thd->query_plan_flags= QPLAN_INIT; > + thd->query_plan_fsort_passes= 0; > + } This part of the patch makes me wonder. Isn't this dead code? Just above, the code does this: if (!query) { is_command= TRUE; query= command_name[thd->command].str; query_length= command_name[thd->command].length; } So the above part of the patch will only run when query!=0 && query_length==0. Can this ever happen? If not, the code should probably be removed. If so, maybe add a comment explaining when this happens (and change it to be in an else {} clause of the if statement just before it). - Kristian.

1 0

[Maria-developers] No more compiler warnings in lp:maria!
by Kristian Nielsen 07 Sep '09

07 Sep '09

While the subject of this mail is not quite true yet, I'm happy to announce progress on this project that Monty and I have been discussing for some time. Basically, compiler warnings are very useful to catch a number of bugs in our sources. This has been proven time and time again. However, to do this effectively, we need to keep a state where there are no compiler warnings in the main tree, so that we can detect any new warnings introduced by patches and fix any problems accordingly. With my latest push this morning, the Buildbot compiles on builders hardy-amd64-valgrind and work-amd64-valgrind are now completely free of warnings. This means any new patches must be also clean from warnings on these hosts. Any newly introduced warnings should be fixed ASAP. There are still some compiler warnings left on the other hosts, which would be really good to also eliminate. Any help in this direction would be greatly appreciated. There are a few compiler warnings that could not be fixed meaningfully in the source for various reasons. These warnings are instead suppressed from Buildbot using the file support-files/compiler_warnings.supp. Over time we should strive to remove things from this file. However, we should not make stupid changes to the code just to make a silly compiler happy. I also added to the tree this script: support-files/ccfilter This can be used locally by developers to apply support-files/compiler_warnings.supp to the output of a local build, just like Buildbot suppresses compiler warnings from this file. This is useful to check that new patches are warning-free before pushing. There are some comments at the start of the script describing usage. But basically, run ./configure with CC="ccfilter gcc" CXX="ccfilter gcc" Then the suppressed compiler warnings will be removed from the compiler output. And setting the FAILONWARNING environment variable can be used to cause a build failure on non-suppressed warning (similar to -Wall, but heeding support-files/compiler_warnings.supp). - Kristian.

1 0

[Maria-developers] [Branch ~maria-captains/maria/5.1] Rev 2729: Fix compiler warnings.
by noreply＠launchpad.net 07 Sep '09

07 Sep '09

------------------------------------------------------------ revno: 2729 committer: knielsen(a)knielsen-hq.org branch nick: work timestamp: Mon 2009-09-07 09:12:38 +0200 message: Fix compiler warnings. modified: sql/opt_table_elimination.cc -- lp:maria https://code.launchpad.net/~maria-captains/maria/5.1 Your team Maria developers is subscribed to branch lp:maria. To unsubscribe from this branch go to https://code.launchpad.net/~maria-captains/maria/5.1/+edit-subscription.

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2729)
by knielsen＠knielsen-hq.org 07 Sep '09

07 Sep '09

#At lp:maria 2729 knielsen(a)knielsen-hq.org 2009-09-07 Fix compiler warnings. modified: sql/opt_table_elimination.cc === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-09-01 21:41:16 +0000 +++ b/sql/opt_table_elimination.cc 2009-09-07 07:12:38 +0000 @@ -1802,16 +1802,16 @@ void Dep_analysis_context::dbug_print_de eq_mod->expr->print(&str, QT_ORDINARY); if (eq_mod->field) { - fprintf(DBUG_FILE, " equality%d: %s -> %s.%s\n", - eq_mod - equality_mods, + fprintf(DBUG_FILE, " equality%ld: %s -> %s.%s\n", + (long)(eq_mod - equality_mods), str.c_ptr(), eq_mod->field->table->table->alias, eq_mod->field->field->field_name); } else { - fprintf(DBUG_FILE, " equality%d: multi-equality", - eq_mod - equality_mods); + fprintf(DBUG_FILE, " equality%ld: multi-equality", + (long)(eq_mod - equality_mods)); } } fprintf(DBUG_FILE,"\n");

1 0

[Maria-developers] Progress (by Knielsen): mysqlbinlog: remove undesired effects of format description binlog statement (50)
by worklog-noreply＠askmonty.org 07 Sep '09

07 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: mysqlbinlog: remove undesired effects of format description binlog statement CREATION DATE..: Mon, 17 Aug 2009, 14:12 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 50 (http://askmonty.org/worklog/?tid=50) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 2 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Mon, 07 Sep 2009, 10:06)=-=- Research worklog, bugs, and existing server code. Worked 2 hours and estimate 0 hours remain (original estimate increased by 2 hours). -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- Dependency created: 39 now depends on 50 -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.11389 2009-08-17 14:13:05.000000000 +0300 +++ /tmp/wklog.50.new.11389 2009-08-17 14:13:05.000000000 +0300 @@ -1 +1,14 @@ +First item +---------- +AFAIU what needs to be done is: +1. record a source server version (it is in the first binlog event). +2. don't emit a BINLOG statement if the recorded version number is 5.0.x or + below. + +and we'll get a 5.0-applicable binlog. + +Second item +----------- +One question that one needs to sort out before disabling server_id change is +why it was put there in the first place? Should it be always removed? DESCRIPTION: According to complaints in BUG#46640: 1. mysqlbinlog will emit a 5.0-incompatible "format description binlog statement" even when reading a binary log that was produced by 5.0. This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. 2. When one applies "format description binlog statement" at the slave, it will change the slave's server_id when applied. This WL is to fix these issues. HIGH-LEVEL SPECIFICATION: First item ---------- AFAIU what needs to be done is: 1. record a source server version (it is in the first binlog event). 2. don't emit a BINLOG statement if the recorded version number is 5.0.x or below. and we'll get a 5.0-applicable binlog. Second item ----------- One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Progress (by Knielsen): mysqlbinlog: remove undesired effects of format description binlog statement (50)
by worklog-noreply＠askmonty.org 07 Sep '09

07 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: mysqlbinlog: remove undesired effects of format description binlog statement CREATION DATE..: Mon, 17 Aug 2009, 14:12 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 50 (http://askmonty.org/worklog/?tid=50) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 2 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Mon, 07 Sep 2009, 10:06)=-=- Research worklog, bugs, and existing server code. Worked 2 hours and estimate 0 hours remain (original estimate increased by 2 hours). -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- Dependency created: 39 now depends on 50 -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.11389 2009-08-17 14:13:05.000000000 +0300 +++ /tmp/wklog.50.new.11389 2009-08-17 14:13:05.000000000 +0300 @@ -1 +1,14 @@ +First item +---------- +AFAIU what needs to be done is: +1. record a source server version (it is in the first binlog event). +2. don't emit a BINLOG statement if the recorded version number is 5.0.x or + below. + +and we'll get a 5.0-applicable binlog. + +Second item +----------- +One question that one needs to sort out before disabling server_id change is +why it was put there in the first place? Should it be always removed? DESCRIPTION: According to complaints in BUG#46640: 1. mysqlbinlog will emit a 5.0-incompatible "format description binlog statement" even when reading a binary log that was produced by 5.0. This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. 2. When one applies "format description binlog statement" at the slave, it will change the slave's server_id when applied. This WL is to fix these issues. HIGH-LEVEL SPECIFICATION: First item ---------- AFAIU what needs to be done is: 1. record a source server version (it is in the first binlog event). 2. don't emit a BINLOG statement if the recorded version number is 5.0.x or below. and we'll get a 5.0-applicable binlog. Second item ----------- One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Documentation for new features in MariaDB
by Kristian Nielsen 07 Sep '09

07 Sep '09

We are starting to add new features to MariaDB (that are not in MySQL), including user-visible features. I think we need to decide now how we will handle keeping documentation for these new features. There is an open issue about the MySQL documentation being non-free, and whether we can do anything about it. But until this is resolved, on way or the other, we still need somewhere to write documentation. I think it is mainly about setting up a repository somewhere and start adding text. The important thing now is to get the content there, the presentation can be enhanced later. So we would have a manual "differences between MariaDB and MySQL" or something. If we have the content we can use it later in whatever form we need, if we omit adding the content now it is going to be really hard to figure out everything that's missing in a year or two. Daniel, any suggestions for what format to use? Can we use the same format as the MySQL documentation (is it docbook or something, can't remember)? Also, until we get procedures like at MySQL/Sun for a documentation team that ensures all features and bug fixes get documented, I think it is the responsibility of each developer that documentation gets added in the right place. I would really like to see us keeping the up the standard that everything in MySQL (now MariaDB) is documented. - Kristian.

7 22

[Maria-developers] [Branch ~maria-captains/maria/5.1] Rev 2728: Fix ha_maria to compile on Windows
by noreply＠launchpad.net 06 Sep '09

06 Sep '09

------------------------------------------------------------ revno: 2728 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-fixwindows timestamp: Sun 2009-09-06 19:49:40 +0400 message: Fix ha_maria to compile on Windows modified: storage/maria/ma_rt_index.c -- lp:maria https://code.launchpad.net/~maria-captains/maria/5.1 Your team Maria developers is subscribed to branch lp:maria. To unsubscribe from this branch go to https://code.launchpad.net/~maria-captains/maria/5.1/+edit-subscription.

1 0

[Maria-developers] Rev 2728: Fix ha_maria to compile on Windows in file:///home/psergey/dev/maria-5.1-fixwindows/
by Sergey Petrunya 06 Sep '09

06 Sep '09

At file:///home/psergey/dev/maria-5.1-fixwindows/ ------------------------------------------------------------ revno: 2728 revision-id: psergey(a)askmonty.org-20090906154940-bdwpqtizq0v9wb4q parent: knielsen(a)knielsen-hq.org-20090904074534-tyya4rfwjgvbt5ch committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-fixwindows timestamp: Sun 2009-09-06 19:49:40 +0400 message: Fix ha_maria to compile on Windows === modified file 'storage/maria/ma_rt_index.c' --- a/storage/maria/ma_rt_index.c 2009-02-19 09:01:25 +0000 +++ b/storage/maria/ma_rt_index.c 2009-09-06 15:49:40 +0000 @@ -1133,13 +1133,13 @@ { uint nod_flag; ulong i; + uchar *page_buf; + MARIA_PAGE page; MARIA_KEY tmp_key; tmp_key.keyinfo= key->keyinfo; tmp_key.data_length= key->data_length; tmp_key.ref_length= key->ref_length; tmp_key.flag= 0; /* Safety */ - uchar *page_buf; - MARIA_PAGE page; if (ReinsertList.n_pages) {

1 0

[Maria-developers] Rev 2727: Fix ha_maria compile error on Windows in file:///home/psergey/dev/maria-5.1-fixwindows/
by Sergey Petrunya 06 Sep '09

06 Sep '09

At file:///home/psergey/dev/maria-5.1-fixwindows/ ------------------------------------------------------------ revno: 2727 revision-id: psergey(a)askmonty.org-20090906154349-y4de6ul0pljlr4yh parent: monty(a)mysql.com-20090903145646-0cis6dfct2c8gee7 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-fixwindows timestamp: Sun 2009-09-06 19:43:49 +0400 message: Fix ha_maria compile error on Windows === modified file 'storage/maria/ma_rt_index.c' --- a/storage/maria/ma_rt_index.c 2009-02-19 09:01:25 +0000 +++ b/storage/maria/ma_rt_index.c 2009-09-06 15:43:49 +0000 @@ -1133,13 +1133,13 @@ { uint nod_flag; ulong i; + uchar *page_buf; + MARIA_PAGE page; MARIA_KEY tmp_key; tmp_key.keyinfo= key->keyinfo; tmp_key.data_length= key->data_length; tmp_key.ref_length= key->ref_length; tmp_key.flag= 0; /* Safety */ - uchar *page_buf; - MARIA_PAGE page; if (ReinsertList.n_pages) {

1 0

[Maria-developers] [Branch ~maria-captains/maria/5.1] Rev 2727: Merge.
by noreply＠launchpad.net 05 Sep '09

05 Sep '09

Merge authors: Kristian Nielsen (knielsen) Paul McCullagh (paul-mccullagh) Related merge proposals: https://code.launchpad.net/~paul-mccullagh/maria/bug-422550/+merge/11076 proposed by: Paul McCullagh (paul-mccullagh) review: Approve - Kristian Nielsen (knielsen) ------------------------------------------------------------ revno: 2727 [merge] committer: knielsen(a)knielsen-hq.org branch nick: work timestamp: Fri 2009-09-04 09:45:34 +0200 message: Merge. - PBXT fixes. - MBug#423035 fix. - Compiler warning fixes. added: mysql-test/r/mysqld_option_err.result mysql-test/t/mysqld_option_err.test support-files/ccfilter modified: client/mysqlbinlog.cc config/ac-macros/misc.m4 extra/yassl/include/yassl_int.hpp extra/yassl/src/handshake.cpp extra/yassl/src/yassl_imp.cpp extra/yassl/src/yassl_int.cpp extra/yassl/taocrypt/include/modes.hpp extra/yassl/taocrypt/src/asn.cpp mysql-test/mysql-test-run.pl mysys/my_compress.c mysys/my_getopt.c sql/mysqld.cc storage/pbxt/ChangeLog storage/pbxt/src/Makefile.am storage/pbxt/src/datalog_xt.cc storage/pbxt/src/discover_xt.cc storage/pbxt/src/filesys_xt.cc storage/pbxt/src/ha_pbxt.cc storage/pbxt/src/memory_xt.cc storage/pbxt/src/myxt_xt.cc storage/pbxt/src/myxt_xt.h storage/pbxt/src/restart_xt.cc storage/pbxt/src/restart_xt.h storage/pbxt/src/strutil_xt.cc storage/pbxt/src/tabcache_xt.cc storage/pbxt/src/thread_xt.cc storage/pbxt/src/xaction_xt.cc storage/pbxt/src/xactlog_xt.cc storage/pbxt/src/xt_errno.h storage/xtradb/fil/fil0fil.c storage/xtradb/mtr/mtr0mtr.c storage/xtradb/srv/srv0srv.c storage/xtradb/srv/srv0start.c strings/decimal.c support-files/compiler_warnings.supp -- lp:maria https://code.launchpad.net/~maria-captains/maria/5.1 Your team Maria developers is subscribed to branch lp:maria. To unsubscribe from this branch go to https://code.launchpad.net/~maria-captains/maria/5.1/+edit-subscription.

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2725)
by knielsen＠knielsen-hq.org 04 Sep '09

04 Sep '09

#At lp:maria 2725 knielsen(a)knielsen-hq.org 2009-09-03 MBug#423035: error in parsing enum value for plugin variable in mysqld command-line option Fix parsing of invalid plugin enum option value. Previous patch to fix plugin enum option parsing on big-endian introduced another bug due to incorrect comparison of unsigned value. This would cause an incorrect value to be parsed as value 0. See also MySQL Bug#41010 and Bug#32034. added: mysql-test/r/mysqld_option_err.result mysql-test/t/mysqld_option_err.test modified: mysql-test/mysql-test-run.pl mysys/my_getopt.c per-file messages: mysql-test/mysql-test-run.pl Add a facility for test case to run the mysqld binary (to test that invalid startup options are rejected correctly). mysql-test/r/mysqld_option_err.result Add a test case to check that invalid startup options for mysqld are rejected. This is needed to test MBug#423035. Also add a few other similar tests, as this was completely untested before this patch. mysql-test/t/mysqld_option_err.test Add a test case to check that invalid startup options for mysqld are rejected. This is needed to test MBug#423035. Also add a few other similar tests, as this was completely untested before this patch. mysys/my_getopt.c Fix parsing of invalid plugin enum option value. === modified file 'mysql-test/mysql-test-run.pl' --- a/mysql-test/mysql-test-run.pl 2009-06-22 08:06:35 +0000 +++ b/mysql-test/mysql-test-run.pl 2009-09-03 13:05:02 +0000 @@ -1771,6 +1771,20 @@ sub tool_arguments ($$) { return mtr_args2str($exe, @$args); } +# This is not used to actually start a mysqld server, just to allow test +# scripts to run the mysqld binary to test invalid server startup options. +sub mysqld_client_arguments () { + my $default_mysqld= default_mysqld(); + my $exe = find_mysqld($basedir); + my $args; + mtr_init_args(\$args); + mtr_add_arg($args, "--no-defaults"); + mtr_add_arg($args, "--basedir=%s", $basedir); + mtr_add_arg($args, "--character-sets-dir=%s", $default_mysqld->value("character-sets-dir")); + mtr_add_arg($args, "--language=%s", $default_mysqld->value("language")); + return mtr_args2str($exe, @$args); +} + sub have_maria_support () { my $maria_var= $mysqld_variables{'maria'}; @@ -1968,6 +1982,7 @@ sub environment_setup { $ENV{'MYSQLADMIN'}= native_path($exe_mysqladmin); $ENV{'MYSQL_CLIENT_TEST'}= mysql_client_test_arguments(); $ENV{'MYSQL_FIX_SYSTEM_TABLES'}= mysql_fix_arguments(); + $ENV{'MYSQLD'}= mysqld_client_arguments(); $ENV{'EXE_MYSQL'}= $exe_mysql; # ---------------------------------------------------- === added file 'mysql-test/r/mysqld_option_err.result' --- a/mysql-test/r/mysqld_option_err.result 1970-01-01 00:00:00 +0000 +++ b/mysql-test/r/mysqld_option_err.result 2009-09-03 13:05:02 +0000 @@ -0,0 +1,6 @@ +Test that unknown option is not silently ignored. +Test bad binlog format. +Test bad default storage engine. +Test non-numeric value passed to number option. +Test that bad value for plugin enum option is rejected correctly. +Done. === added file 'mysql-test/t/mysqld_option_err.test' --- a/mysql-test/t/mysqld_option_err.test 1970-01-01 00:00:00 +0000 +++ b/mysql-test/t/mysqld_option_err.test 2009-09-03 13:05:02 +0000 @@ -0,0 +1,47 @@ +# +# Test error checks on mysqld command line option parsing. +# +# Call mysqld with different invalid options, and check that it fails in each case. +# +# This means that a test failure results in mysqld starting up, which is only +# caught when the test case times out. This is not ideal, but I did not find an +# easy way to have the server shut down after a successful startup. +# + +--source include/not_embedded.inc + +# We have not run (and do not need) bootstrap of the server. We just +# give it a dummy data directory (for log files etc). + +mkdir $MYSQLTEST_VARDIR/tmp/mysqld_option_err; + + +--echo Test that unknown option is not silently ignored. +--error 2 +--exec $MYSQLD --skip-networking --datadir=$MYSQLTEST_VARDIR/tmp/mysqld_option_err --skip-grant-tables --nonexistentoption >$MYSQLTEST_VARDIR/tmp/mysqld_option_err/mysqltest.log 2>&1 + + +--echo Test bad binlog format. +--error 1 +--exec $MYSQLD --skip-networking --datadir=$MYSQLTEST_VARDIR/tmp/mysqld_option_err --skip-grant-tables --log-bin --binlog-format=badformat >>$MYSQLTEST_VARDIR/tmp/mysqld_option_err/mysqltest.log 2>&1 + + +--echo Test bad default storage engine. +--error 1 +--exec $MYSQLD --skip-networking --datadir=$MYSQLTEST_VARDIR/tmp/mysqld_option_err --skip-grant-tables --default-storage-engine=nonexistentengine >>$MYSQLTEST_VARDIR/tmp/mysqld_option_err/mysqltest.log 2>&1 + + +--echo Test non-numeric value passed to number option. +--error 1 +--exec $MYSQLD --skip-networking --datadir=$MYSQLTEST_VARDIR/tmp/mysqld_option_err --skip-grant-tables --min-examined-row-limit=notanumber >>$MYSQLTEST_VARDIR/tmp/mysqld_option_err/mysqltest.log 2>&1 + + +# Test for MBug#423035: error in parsing enum value for plugin +# variable in mysqld command-line option. +# See also Bug#32034. +--echo Test that bad value for plugin enum option is rejected correctly. +--error 7 +--exec $MYSQLD --skip-networking --datadir=$MYSQLTEST_VARDIR/tmp/mysqld_option_err --skip-grant-tables $EXAMPLE_PLUGIN_OPT --plugin-load=EXAMPLE=ha_example.so --plugin-example-enum-var=noexist >>$MYSQLTEST_VARDIR/tmp/mysqld_option_err/mysqltest.log 2>&1 + + +--echo Done. === modified file 'mysys/my_getopt.c' --- a/mysys/my_getopt.c 2009-05-20 15:34:34 +0000 +++ b/mysys/my_getopt.c 2009-09-03 13:05:02 +0000 @@ -603,6 +603,7 @@ static int setval(const struct my_option my_bool set_maximum_value) { int err= 0; + int pos; if (value && argument) { @@ -647,7 +648,9 @@ static int setval(const struct my_option return EXIT_OUT_OF_MEMORY; break; case GET_ENUM: - if (((*(ulong *)result_pos)= find_type(argument, opts->typelib, 2) - 1) < 0) + pos= find_type(argument, opts->typelib, 2) - 1; + (*(ulong *)result_pos)= pos; + if (pos < 0) return EXIT_ARGUMENT_INVALID; break; case GET_SET:

2 1

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2727)
by knielsen＠knielsen-hq.org 04 Sep '09

04 Sep '09

#At lp:maria 2727 knielsen(a)knielsen-hq.org 2009-09-04 [merge] Merge. - PBXT fixes. - MBug#423035 fix. - Compiler warning fixes. added: mysql-test/r/mysqld_option_err.result mysql-test/t/mysqld_option_err.test support-files/ccfilter modified: client/mysqlbinlog.cc config/ac-macros/misc.m4 extra/yassl/include/yassl_int.hpp extra/yassl/src/handshake.cpp extra/yassl/src/yassl_imp.cpp extra/yassl/src/yassl_int.cpp extra/yassl/taocrypt/include/modes.hpp extra/yassl/taocrypt/src/asn.cpp mysql-test/mysql-test-run.pl mysys/my_compress.c mysys/my_getopt.c sql/mysqld.cc storage/pbxt/ChangeLog storage/pbxt/src/Makefile.am storage/pbxt/src/datalog_xt.cc storage/pbxt/src/discover_xt.cc storage/pbxt/src/filesys_xt.cc storage/pbxt/src/ha_pbxt.cc storage/pbxt/src/memory_xt.cc storage/pbxt/src/myxt_xt.cc storage/pbxt/src/myxt_xt.h storage/pbxt/src/restart_xt.cc storage/pbxt/src/restart_xt.h storage/pbxt/src/strutil_xt.cc storage/pbxt/src/tabcache_xt.cc storage/pbxt/src/thread_xt.cc storage/pbxt/src/xaction_xt.cc storage/pbxt/src/xactlog_xt.cc storage/pbxt/src/xt_errno.h storage/xtradb/fil/fil0fil.c storage/xtradb/mtr/mtr0mtr.c storage/xtradb/srv/srv0srv.c storage/xtradb/srv/srv0start.c strings/decimal.c support-files/compiler_warnings.supp === modified file 'client/mysqlbinlog.cc' --- a/client/mysqlbinlog.cc 2009-02-13 16:41:47 +0000 +++ b/client/mysqlbinlog.cc 2009-09-03 13:20:22 +0000 @@ -689,8 +689,8 @@ Exit_status process_event(PRINT_EVENT_IN Format events are not concerned by --offset and such, we always need to read them to be able to process the wanted events. */ - if ((rec_count >= offset) && - ((my_time_t)(ev->when) >= start_datetime) || + if (((rec_count >= offset) && + ((my_time_t)(ev->when) >= start_datetime)) || (ev_type == FORMAT_DESCRIPTION_EVENT)) { if (ev_type != FORMAT_DESCRIPTION_EVENT) === modified file 'config/ac-macros/misc.m4' --- a/config/ac-macros/misc.m4 2008-02-26 17:38:43 +0000 +++ b/config/ac-macros/misc.m4 2009-09-03 13:20:22 +0000 @@ -594,15 +594,15 @@ dnl ------------------------------------ dnl MYSQL_NEEDS_MYSYS_NEW AC_DEFUN([MYSQL_NEEDS_MYSYS_NEW], -[AC_CACHE_CHECK([needs mysys_new helpers], mysql_use_mysys_new, +[AC_CACHE_CHECK([needs mysys_new helpers], mysql_cv_use_mysys_new, [ AC_LANG_PUSH(C++) AC_TRY_LINK([], [ class A { public: int b; }; A *a=new A; a->b=10; delete a; -], mysql_use_mysys_new=no, mysql_use_mysys_new=yes) +], mysql_cv_use_mysys_new=no, mysql_cv_use_mysys_new=yes) AC_LANG_POP(C++) ]) -if test "$mysql_use_mysys_new" = "yes" +if test "$mysql_cv_use_mysys_new" = "yes" then AC_DEFINE([USE_MYSYS_NEW], [1], [Needs to use mysys_new helpers]) fi === modified file 'extra/yassl/include/yassl_int.hpp' --- a/extra/yassl/include/yassl_int.hpp 2008-11-18 16:45:44 +0000 +++ b/extra/yassl/include/yassl_int.hpp 2009-09-03 13:20:22 +0000 @@ -441,7 +441,7 @@ public: const Ciphers& GetCiphers() const; const DH_Parms& GetDH_Parms() const; const Stats& GetStats() const; - const VerifyCallback getVerifyCallback() const; + VerifyCallback getVerifyCallback() const; pem_password_cb GetPasswordCb() const; void* GetUserData() const; bool GetSessionCacheOff() const; === modified file 'extra/yassl/src/handshake.cpp' --- a/extra/yassl/src/handshake.cpp 2008-04-28 16:24:05 +0000 +++ b/extra/yassl/src/handshake.cpp 2009-09-03 13:20:22 +0000 @@ -789,7 +789,7 @@ void processReply(SSL& ssl) { if (ssl.GetError()) return; - if (DoProcessReply(ssl)) + if (DoProcessReply(ssl)) { // didn't complete process if (!ssl.getSocket().IsNonBlocking()) { // keep trying now, blocking ok @@ -799,6 +799,7 @@ void processReply(SSL& ssl) else // user will have try again later, non blocking ssl.SetError(YasslError(SSL_ERROR_WANT_READ)); + } } @@ -872,11 +873,12 @@ void sendServerKeyExchange(SSL& ssl, Buf // send change cipher void sendChangeCipher(SSL& ssl, BufferOutput buffer) { - if (ssl.getSecurity().get_parms().entity_ == server_end) + if (ssl.getSecurity().get_parms().entity_ == server_end) { if (ssl.getSecurity().get_resuming()) ssl.verifyState(clientKeyExchangeComplete); else ssl.verifyState(clientFinishedComplete); + } if (ssl.GetError()) return; ChangeCipherSpec ccs; === modified file 'extra/yassl/src/yassl_imp.cpp' --- a/extra/yassl/src/yassl_imp.cpp 2009-02-13 16:41:47 +0000 +++ b/extra/yassl/src/yassl_imp.cpp 2009-09-03 13:20:22 +0000 @@ -1304,7 +1304,7 @@ void ServerHello::Process(input_buffer&, else ssl.useSecurity().use_connection().sessionID_Set_ = false; - if (ssl.getSecurity().get_resuming()) + if (ssl.getSecurity().get_resuming()) { if (memcmp(session_id_, ssl.getSecurity().get_resume().GetID(), ID_LEN) == 0) { ssl.set_masterSecret(ssl.getSecurity().get_resume().GetSecret()); @@ -1319,6 +1319,7 @@ void ServerHello::Process(input_buffer&, ssl.useSecurity().set_resuming(false); ssl.useLog().Trace("server denied resumption"); } + } if (ssl.CompressionOn() && !compression_method_) ssl.UnSetCompression(); // server isn't supporting yaSSL zlib request === modified file 'extra/yassl/src/yassl_int.cpp' --- a/extra/yassl/src/yassl_int.cpp 2009-02-13 16:41:47 +0000 +++ b/extra/yassl/src/yassl_int.cpp 2009-09-03 13:20:22 +0000 @@ -1833,7 +1833,7 @@ SSL_CTX::GetCA_List() const } -const VerifyCallback SSL_CTX::getVerifyCallback() const +VerifyCallback SSL_CTX::getVerifyCallback() const { return verifyCallback_; } === modified file 'extra/yassl/taocrypt/include/modes.hpp' --- a/extra/yassl/taocrypt/include/modes.hpp 2007-03-23 12:43:09 +0000 +++ b/extra/yassl/taocrypt/include/modes.hpp 2009-09-03 13:20:22 +0000 @@ -95,11 +95,12 @@ inline void Mode_BASE::Process(byte* out { if (mode_ == ECB) ECB_Process(out, in, sz); - else if (mode_ == CBC) + else if (mode_ == CBC) { if (dir_ == ENCRYPTION) CBC_Encrypt(out, in, sz); else CBC_Decrypt(out, in, sz); + } } === modified file 'extra/yassl/taocrypt/src/asn.cpp' --- a/extra/yassl/taocrypt/src/asn.cpp 2009-02-13 16:41:47 +0000 +++ b/extra/yassl/taocrypt/src/asn.cpp 2009-09-03 13:20:22 +0000 @@ -780,11 +780,12 @@ void CertDecoder::GetDate(DateType dt) memcpy(date, source_.get_current(), length); source_.advance(length); - if (!ValidateDate(date, b, dt) && verify_) + if (!ValidateDate(date, b, dt) && verify_) { if (dt == BEFORE) source_.SetError(BEFORE_DATE_E); else source_.SetError(AFTER_DATE_E); + } // save for later use if (dt == BEFORE) { @@ -1061,7 +1062,7 @@ word32 DecodeDSA_Signature(byte* decoded return 0; } word32 rLen = GetLength(source); - if (rLen != 20) + if (rLen != 20) { if (rLen == 21) { // zero at front, eat source.next(); --rLen; @@ -1074,6 +1075,7 @@ word32 DecodeDSA_Signature(byte* decoded source.SetError(DSA_SZ_E); return 0; } + } memcpy(decoded, source.get_buffer() + source.get_index(), rLen); source.advance(rLen); @@ -1083,7 +1085,7 @@ word32 DecodeDSA_Signature(byte* decoded return 0; } word32 sLen = GetLength(source); - if (sLen != 20) + if (sLen != 20) { if (sLen == 21) { source.next(); // zero at front, eat --sLen; @@ -1096,6 +1098,7 @@ word32 DecodeDSA_Signature(byte* decoded source.SetError(DSA_SZ_E); return 0; } + } memcpy(decoded + rLen, source.get_buffer() + source.get_index(), sLen); source.advance(sLen); === modified file 'mysql-test/mysql-test-run.pl' --- a/mysql-test/mysql-test-run.pl 2009-08-25 15:02:55 +0000 +++ b/mysql-test/mysql-test-run.pl 2009-09-04 07:45:34 +0000 @@ -1773,6 +1773,20 @@ sub tool_arguments ($$) { return mtr_args2str($exe, @$args); } +# This is not used to actually start a mysqld server, just to allow test +# scripts to run the mysqld binary to test invalid server startup options. +sub mysqld_client_arguments () { + my $default_mysqld= default_mysqld(); + my $exe = find_mysqld($basedir); + my $args; + mtr_init_args(\$args); + mtr_add_arg($args, "--no-defaults"); + mtr_add_arg($args, "--basedir=%s", $basedir); + mtr_add_arg($args, "--character-sets-dir=%s", $default_mysqld->value("character-sets-dir")); + mtr_add_arg($args, "--language=%s", $default_mysqld->value("language")); + return mtr_args2str($exe, @$args); +} + sub have_maria_support () { my $maria_var= $mysqld_variables{'maria'}; @@ -1970,6 +1984,7 @@ sub environment_setup { $ENV{'MYSQLADMIN'}= native_path($exe_mysqladmin); $ENV{'MYSQL_CLIENT_TEST'}= mysql_client_test_arguments(); $ENV{'MYSQL_FIX_SYSTEM_TABLES'}= mysql_fix_arguments(); + $ENV{'MYSQLD'}= mysqld_client_arguments(); $ENV{'EXE_MYSQL'}= $exe_mysql; # ---------------------------------------------------- === added file 'mysql-test/r/mysqld_option_err.result' --- a/mysql-test/r/mysqld_option_err.result 1970-01-01 00:00:00 +0000 +++ b/mysql-test/r/mysqld_option_err.result 2009-09-03 13:05:02 +0000 @@ -0,0 +1,6 @@ +Test that unknown option is not silently ignored. +Test bad binlog format. +Test bad default storage engine. +Test non-numeric value passed to number option. +Test that bad value for plugin enum option is rejected correctly. +Done. === added file 'mysql-test/t/mysqld_option_err.test' --- a/mysql-test/t/mysqld_option_err.test 1970-01-01 00:00:00 +0000 +++ b/mysql-test/t/mysqld_option_err.test 2009-09-03 13:05:02 +0000 @@ -0,0 +1,47 @@ +# +# Test error checks on mysqld command line option parsing. +# +# Call mysqld with different invalid options, and check that it fails in each case. +# +# This means that a test failure results in mysqld starting up, which is only +# caught when the test case times out. This is not ideal, but I did not find an +# easy way to have the server shut down after a successful startup. +# + +--source include/not_embedded.inc + +# We have not run (and do not need) bootstrap of the server. We just +# give it a dummy data directory (for log files etc). + +mkdir $MYSQLTEST_VARDIR/tmp/mysqld_option_err; + + +--echo Test that unknown option is not silently ignored. +--error 2 +--exec $MYSQLD --skip-networking --datadir=$MYSQLTEST_VARDIR/tmp/mysqld_option_err --skip-grant-tables --nonexistentoption >$MYSQLTEST_VARDIR/tmp/mysqld_option_err/mysqltest.log 2>&1 + + +--echo Test bad binlog format. +--error 1 +--exec $MYSQLD --skip-networking --datadir=$MYSQLTEST_VARDIR/tmp/mysqld_option_err --skip-grant-tables --log-bin --binlog-format=badformat >>$MYSQLTEST_VARDIR/tmp/mysqld_option_err/mysqltest.log 2>&1 + + +--echo Test bad default storage engine. +--error 1 +--exec $MYSQLD --skip-networking --datadir=$MYSQLTEST_VARDIR/tmp/mysqld_option_err --skip-grant-tables --default-storage-engine=nonexistentengine >>$MYSQLTEST_VARDIR/tmp/mysqld_option_err/mysqltest.log 2>&1 + + +--echo Test non-numeric value passed to number option. +--error 1 +--exec $MYSQLD --skip-networking --datadir=$MYSQLTEST_VARDIR/tmp/mysqld_option_err --skip-grant-tables --min-examined-row-limit=notanumber >>$MYSQLTEST_VARDIR/tmp/mysqld_option_err/mysqltest.log 2>&1 + + +# Test for MBug#423035: error in parsing enum value for plugin +# variable in mysqld command-line option. +# See also Bug#32034. +--echo Test that bad value for plugin enum option is rejected correctly. +--error 7 +--exec $MYSQLD --skip-networking --datadir=$MYSQLTEST_VARDIR/tmp/mysqld_option_err --skip-grant-tables $EXAMPLE_PLUGIN_OPT --plugin-load=EXAMPLE=ha_example.so --plugin-example-enum-var=noexist >>$MYSQLTEST_VARDIR/tmp/mysqld_option_err/mysqltest.log 2>&1 + + +--echo Done. === modified file 'mysys/my_compress.c' --- a/mysys/my_compress.c 2009-05-22 12:38:50 +0000 +++ b/mysys/my_compress.c 2009-09-03 13:20:22 +0000 @@ -81,12 +81,13 @@ my_bool my_compress(uchar *packet, size_ This fix is safe, since such memory is only used internally by zlib, so we will not hide any bugs in mysql this way. */ -void *my_az_allocator(void *dummy, unsigned int items, unsigned int size) +void *my_az_allocator(void *dummy __attribute__((unused)), unsigned int items, + unsigned int size) { return my_malloc((size_t)items*(size_t)size, IF_VALGRIND(MY_ZEROFILL, MYF(0))); } -void my_az_free(void *dummy, void *address) +void my_az_free(void *dummy __attribute__((unused)), void *address) { my_free(address, MYF(MY_ALLOW_ZERO_PTR)); } === modified file 'mysys/my_getopt.c' --- a/mysys/my_getopt.c 2009-05-20 15:34:34 +0000 +++ b/mysys/my_getopt.c 2009-09-03 13:05:02 +0000 @@ -603,6 +603,7 @@ static int setval(const struct my_option my_bool set_maximum_value) { int err= 0; + int pos; if (value && argument) { @@ -647,7 +648,9 @@ static int setval(const struct my_option return EXIT_OUT_OF_MEMORY; break; case GET_ENUM: - if (((*(ulong *)result_pos)= find_type(argument, opts->typelib, 2) - 1) < 0) + pos= find_type(argument, opts->typelib, 2) - 1; + (*(ulong *)result_pos)= pos; + if (pos < 0) return EXIT_ARGUMENT_INVALID; break; case GET_SET: === modified file 'sql/mysqld.cc' --- a/sql/mysqld.cc 2009-09-03 14:56:46 +0000 +++ b/sql/mysqld.cc 2009-09-04 07:45:34 +0000 @@ -1016,6 +1016,7 @@ static void close_connections(void) } +#ifdef HAVE_CLOSE_SERVER_SOCK static void close_socket(my_socket sock, const char *info) { DBUG_ENTER("close_socket"); @@ -1035,6 +1036,7 @@ static void close_socket(my_socket sock, } DBUG_VOID_RETURN; } +#endif static void close_server_sock() === modified file 'storage/pbxt/ChangeLog' --- a/storage/pbxt/ChangeLog 2009-08-18 07:46:53 +0000 +++ b/storage/pbxt/ChangeLog 2009-09-03 06:15:03 +0000 @@ -1,6 +1,10 @@ PBXT Release Notes ================== +------- 1.0.08d RC2 - 2009-09-02 + +RN267: Fixed a bug that caused MySQL to crash on shutdown, after an incorrect command line parameter was given. The crash occurred because the background recovery task was not cleaned up before the PBXT engine was de-initialized. + ------- 1.0.08c RC2 - 2009-08-18 RN266: Updated BLOB streaming glue, used with the PBMS engine. The glue code is now identical to the version of "1.0.08-rc-pbms" version of PBXT available from http://blobstreaming.org/download. === modified file 'storage/pbxt/src/Makefile.am' --- a/storage/pbxt/src/Makefile.am 2009-08-31 11:07:44 +0000 +++ b/storage/pbxt/src/Makefile.am 2009-09-04 07:29:34 +0000 @@ -30,7 +30,7 @@ libpbxt_la_SOURCES = bsearch_xt.cc cache datadic_xt.cc datalog_xt.cc filesys_xt.cc hashtab_xt.cc \ ha_pbxt.cc heap_xt.cc index_xt.cc linklist_xt.cc \ memory_xt.cc myxt_xt.cc pthread_xt.cc restart_xt.cc \ - pbms_enabled.cc sortedlist_xt.cc strutil_xt.cc \ + sortedlist_xt.cc strutil_xt.cc \ tabcache_xt.cc table_xt.cc trace_xt.cc thread_xt.cc \ systab_xt.cc ha_xtsys.cc discover_xt.cc \ util_xt.cc xaction_xt.cc xactlog_xt.cc lock_xt.cc locklist_xt.cc @@ -49,4 +49,4 @@ libpbxt_a_SOURCES = $(libpbxt_la_SOURCE libpbxt_a_CXXFLAGS = $(AM_CXXFLAGS) libpbxt_a_CFLAGS = $(AM_CFLAGS) -std=c99 -EXTRA_DIST = CMakeLists.txt +EXTRA_DIST = CMakeLists.txt pbms_enabled.cc === modified file 'storage/pbxt/src/datalog_xt.cc' --- a/storage/pbxt/src/datalog_xt.cc 2009-08-18 07:46:53 +0000 +++ b/storage/pbxt/src/datalog_xt.cc 2009-09-04 07:29:34 +0000 @@ -410,7 +410,7 @@ static void dl_recover_log(XTThreadPtr s ASSERT_NS(seq_read.sl_log_eof == seq_read.sl_rec_log_offset); data_log->dlf_log_eof = seq_read.sl_rec_log_offset; - if (data_log->dlf_log_eof < sizeof(XTXactLogHeaderDRec)) { + if ((size_t) data_log->dlf_log_eof < sizeof(XTXactLogHeaderDRec)) { data_log->dlf_log_eof = sizeof(XTXactLogHeaderDRec); if (!dl_create_log_header(data_log, seq_read.sl_log_file, self)) xt_throw(self); @@ -2015,7 +2015,8 @@ static void *dl_run_co_thread(XTThreadPt int count; void *mysql_thread; - mysql_thread = myxt_create_thread(); + if (!(mysql_thread = myxt_create_thread())) + xt_throw(self); while (!self->t_quit) { try_(a) { @@ -2068,7 +2069,10 @@ static void *dl_run_co_thread(XTThreadPt } } + /* + * {MYSQL-THREAD-KILL} myxt_destroy_thread(mysql_thread, TRUE); + */ return NULL; } === modified file 'storage/pbxt/src/discover_xt.cc' --- a/storage/pbxt/src/discover_xt.cc 2009-08-17 11:12:36 +0000 +++ b/storage/pbxt/src/discover_xt.cc 2009-09-03 13:20:22 +0000 @@ -493,8 +493,8 @@ mysql_prepare_create_table(THD *thd, HA_ } /* Don't pack rows in old tables if the user has requested this */ if ((sql_field->flags & BLOB_FLAG) || - sql_field->sql_type == MYSQL_TYPE_VARCHAR && - create_info->row_type != ROW_TYPE_FIXED) + (sql_field->sql_type == MYSQL_TYPE_VARCHAR && + create_info->row_type != ROW_TYPE_FIXED)) (*db_options)|= HA_OPTION_PACK_RECORD; it2.rewind(); } @@ -963,7 +963,7 @@ mysql_prepare_create_table(THD *thd, HA_ sql_field->sql_type == MYSQL_TYPE_VARCHAR || sql_field->pack_flag & FIELDFLAG_BLOB))) { - if (column_nr == 0 && (sql_field->pack_flag & FIELDFLAG_BLOB) || + if ((column_nr == 0 && (sql_field->pack_flag & FIELDFLAG_BLOB)) || sql_field->sql_type == MYSQL_TYPE_VARCHAR) key_info->flags|= HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY; else === modified file 'storage/pbxt/src/filesys_xt.cc' --- a/storage/pbxt/src/filesys_xt.cc 2009-08-18 07:46:53 +0000 +++ b/storage/pbxt/src/filesys_xt.cc 2009-09-03 06:15:03 +0000 @@ -55,6 +55,7 @@ //#define DEBUG_TRACE_MAP_IO //#define DEBUG_TRACE_FILES //#define INJECT_WRITE_REMAP_ERROR +/* This is required to make testing on the Mac faster: */ #endif #ifdef DEBUG_TRACE_FILES @@ -62,6 +63,10 @@ #define PRINTF xt_trace #endif +#if defined(XT_MAC) && defined(F_FULLFSYNC) +#undef F_FULLFSYNC +#endif + #ifdef INJECT_WRITE_REMAP_ERROR #define INJECT_REMAP_FILE_SIZE 1000000 #define INJECT_REMAP_FILE_TYPE "xtd" === modified file 'storage/pbxt/src/ha_pbxt.cc' --- a/storage/pbxt/src/ha_pbxt.cc 2009-08-18 07:46:53 +0000 +++ b/storage/pbxt/src/ha_pbxt.cc 2009-09-03 06:15:03 +0000 @@ -71,6 +71,7 @@ extern "C" char **session_query(Session #include "tabcache_xt.h" #include "systab_xt.h" #include "xaction_xt.h" +#include "restart_xt.h" #ifdef DEBUG //#define XT_USE_SYS_PAR_DEBUG_SIZES @@ -105,8 +106,6 @@ static void ha_aquire_exclusive_use(XTT static void ha_release_exclusive_use(XTThreadPtr self, XTSharePtr share); static void ha_close_open_tables(XTThreadPtr self, XTSharePtr share, ha_pbxt *mine); -extern void xt_xres_start_database_recovery(XTThreadPtr self); - #ifdef TRACE_STATEMENTS #ifdef PRINT_STATEMENTS @@ -463,8 +462,14 @@ xtPublic void xt_ha_open_database_of_tab if (!self->st_database) { if (!pbxt_database) { xt_open_database(self, mysql_real_data_home, TRUE); - pbxt_database = self->st_database; - xt_heap_reference(self, pbxt_database); + /* {GLOBAL-DB} + * This can be done at the same time as the recovery thread, + * strictly speaking I need a lock. + */ + if (!pbxt_database) { + pbxt_database = self->st_database; + xt_heap_reference(self, pbxt_database); + } } else xt_use_database(self, pbxt_database, XT_FOR_USER); @@ -963,6 +968,8 @@ static void pbxt_call_exit(XTThreadPtr s */ static void ha_exit(XTThreadPtr self) { + xt_xres_wait_for_recovery(self); + /* Wrap things up... */ xt_unuse_database(self, self); /* Just in case the main thread has a database in use (for testing)? */ /* This may cause the streaming engine to cleanup connections and === modified file 'storage/pbxt/src/memory_xt.cc' --- a/storage/pbxt/src/memory_xt.cc 2009-08-17 11:12:36 +0000 +++ b/storage/pbxt/src/memory_xt.cc 2009-09-03 06:15:03 +0000 @@ -34,7 +34,7 @@ #include "trace_xt.h" #ifdef DEBUG -#define RECORD_MM +//#define RECORD_MM #endif #ifdef DEBUG === modified file 'storage/pbxt/src/myxt_xt.cc' --- a/storage/pbxt/src/myxt_xt.cc 2009-08-18 07:46:53 +0000 +++ b/storage/pbxt/src/myxt_xt.cc 2009-09-03 06:15:03 +0000 @@ -1957,7 +1957,7 @@ static TABLE *my_open_table(XTThreadPtr xt_free(self, table); lex_end(&new_lex); thd->lex = old_lex; - xt_throw_ulxterr(XT_CONTEXT, XT_ERR_LOADING_MYSQL_DIC, (u_long) error); + xt_throw_sulxterr(XT_CONTEXT, XT_ERR_LOADING_MYSQL_DIC, tab_path->ps_path, (u_long) error); return NULL; } #else @@ -1975,11 +1975,23 @@ static TABLE *my_open_table(XTThreadPtr #endif #endif + /* If MySQL shutsdown while we are just starting up, they + * they kill the plugin sub-system before calling + * shutdown for the engine! + */ + if (!ha_resolve_by_legacy_type(thd, DB_TYPE_PBXT)) { + xt_free(self, table); + lex_end(&new_lex); + thd->lex = old_lex; + xt_throw_xterr(XT_CONTEXT, XT_ERR_MYSQL_SHUTDOWN); + return NULL; + } + if ((error = open_table_def(thd, share, 0))) { xt_free(self, table); lex_end(&new_lex); thd->lex = old_lex; - xt_throw_ulxterr(XT_CONTEXT, XT_ERR_LOADING_MYSQL_DIC, (u_long) error); + xt_throw_sulxterr(XT_CONTEXT, XT_ERR_LOADING_MYSQL_DIC, tab_path->ps_path, (u_long) error); return NULL; } @@ -1992,7 +2004,7 @@ static TABLE *my_open_table(XTThreadPtr xt_free(self, table); lex_end(&new_lex); thd->lex = old_lex; - xt_throw_ulxterr(XT_CONTEXT, XT_ERR_LOADING_MYSQL_DIC, (u_long) error); + xt_throw_sulxterr(XT_CONTEXT, XT_ERR_LOADING_MYSQL_DIC, tab_path->ps_path, (u_long) error); return NULL; } #endif @@ -2959,6 +2971,46 @@ xtPublic void *myxt_create_thread() return NULL; } + /* + * Unfortunately, if PBXT is the default engine, and we are shutting down + * then global_system_variables.table_plugin may be NULL. Which will cause + * a crash if we try to create a thread! + * + * The following call in plugin_shutdown() sets the global reference + * to NULL: + * + * unlock_variables(NULL, &global_system_variables); + * + * Later plugin_deinitialize() is called. + * + * The following stack is an example crash which occurs when I call + * myxt_create_thread() in ha_exit(), to force the error. + * + * if (pi->state & (PLUGIN_IS_READY | PLUGIN_IS_UNINITIALIZED)) + * pi is NULL! + * #0 0x002ff684 in intern_plugin_lock at sql_plugin.cc:617 + * #1 0x0030296d in plugin_thdvar_init at sql_plugin.cc:2432 + * #2 0x000db4a4 in THD::init at sql_class.cc:756 + * #3 0x000e02ed in THD::THD at sql_class.cc:638 + * #4 0x00e2678d in myxt_create_thread at myxt_xt.cc:2990 + * #5 0x00e05d43 in ha_exit at ha_pbxt.cc:1011 + * #6 0x00e065c2 in pbxt_end at ha_pbxt.cc:1330 + * #7 0x00e065df in pbxt_panic at ha_pbxt.cc:1343 + * #8 0x0023e57d in ha_finalize_handlerton at handler.cc:392 + * #9 0x002ffc8b in plugin_deinitialize at sql_plugin.cc:816 + * #10 0x003037d9 in plugin_shutdown at sql_plugin.cc:1572 + * #11 0x000f7b2b in clean_up at mysqld.cc:1266 + * #12 0x000f7fca in unireg_end at mysqld.cc:1192 + * #13 0x000fa021 in kill_server at mysqld.cc:1134 + * #14 0x000fa6df in kill_server_thread at mysqld.cc:1155 + * #15 0x91fdb155 in _pthread_start + * #16 0x91fdb012 in thread_start + */ + if (!global_system_variables.table_plugin) { + xt_register_xterr(XT_REG_CONTEXT, XT_ERR_MYSQL_NO_THREAD); + return NULL; + } + if (!(new_thd = new THD())) { my_thread_end(); xt_register_error(XT_REG_CONTEXT, XT_ERR_MYSQL_ERROR, 0, "Unable to create MySQL thread (THD)"); @@ -2977,6 +3029,10 @@ xtPublic void *myxt_create_thread() xtPublic void myxt_destroy_thread(void *, xtBool) { } + +xtPublic void myxt_delete_remaining_thread() +{ +} #else xtPublic void myxt_destroy_thread(void *thread, xtBool end_threads) { @@ -3004,6 +3060,14 @@ xtPublic void myxt_destroy_thread(void * if (end_threads) my_thread_end(); } + +xtPublic void myxt_delete_remaining_thread() +{ + THD *thd; + + if ((thd = current_thd)) + myxt_destroy_thread((void *) thd, TRUE); +} #endif xtPublic XTThreadPtr myxt_get_self() === modified file 'storage/pbxt/src/myxt_xt.h' --- a/storage/pbxt/src/myxt_xt.h 2009-08-18 07:46:53 +0000 +++ b/storage/pbxt/src/myxt_xt.h 2009-09-03 06:15:03 +0000 @@ -81,6 +81,7 @@ struct charset_info_st *myxt_getcharset( void *myxt_create_thread(); void myxt_destroy_thread(void *thread, xtBool end_threads); +void myxt_delete_remaining_thread(); XTThreadPtr myxt_get_self(); int myxt_statistics_fill_table(XTThreadPtr self, void *th, void *ta, void *co, MX_CONST void *ch); === modified file 'storage/pbxt/src/restart_xt.cc' --- a/storage/pbxt/src/restart_xt.cc 2009-08-18 07:46:53 +0000 +++ b/storage/pbxt/src/restart_xt.cc 2009-09-03 06:15:03 +0000 @@ -2551,7 +2551,8 @@ static void *xres_cp_run_thread(XTThread int count; void *mysql_thread; - mysql_thread = myxt_create_thread(); + if (!(mysql_thread = myxt_create_thread())) + xt_throw(self); while (!self->t_quit) { try_(a) { @@ -2588,7 +2589,10 @@ static void *xres_cp_run_thread(XTThread } } + /* + * {MYSQL-THREAD-KILL} myxt_destroy_thread(mysql_thread, TRUE); + */ return NULL; } @@ -3179,22 +3183,74 @@ xtPublic void xt_dump_xlogs(XTDatabaseHP * D A T A B A S E R E C O V E R Y T H R E A D */ -extern XTDatabaseHPtr pbxt_database; +extern XTDatabaseHPtr pbxt_database; +static XTThreadPtr xres_recovery_thread; static void *xn_xres_run_recovery_thread(XTThreadPtr self) { THD *mysql_thread; - mysql_thread = (THD *)myxt_create_thread(); + if (!(mysql_thread = (THD *) myxt_create_thread())) + xt_throw(self); - while(!ha_resolve_by_legacy_type(mysql_thread, DB_TYPE_PBXT)) + while (!xres_recovery_thread->t_quit && !ha_resolve_by_legacy_type(mysql_thread, DB_TYPE_PBXT)) xt_sleep_milli_second(1); - xt_open_database(self, mysql_real_data_home, TRUE); - pbxt_database = self->st_database; - xt_heap_reference(self, pbxt_database); + if (!xres_recovery_thread->t_quit) { + /* {GLOBAL-DB} + * It can happen that something will just get in before this + * thread and open/recover the database! + */ + if (!pbxt_database) { + try_(a) { + xt_open_database(self, mysql_real_data_home, TRUE); + /* This can be done at the same time by a foreground thread, + * strictly speaking I need a lock. + */ + if (!pbxt_database) { + pbxt_database = self->st_database; + xt_heap_reference(self, pbxt_database); + } + } + catch_(a) { + xt_log_and_clear_exception(self); + } + cont_(a); + } + } + + /* + * {MYSQL-THREAD-KILL} + * Here is the problem with destroying the thread at this + * point. If we had an error started, then it can lead + * to a callback into pbxt: pbxt_panic(). + * + * This will shutdown things, making it impossible quite the + * thread and do a cleanup. Solution: + * + * Move the MySQL thread descruction to a later point! + * + * sql/mysqld --no-defaults --basedir=~/maria/trunk + * --character-sets-dir=~/maria/trunk/sql/share/charsets + * --language=~/maria/trunk/sql/share/english + * --skip-networking --datadir=/tmp/x --skip-grant-tables --nonexistentoption + * + * #0 0x003893f9 in xt_exit_databases at database_xt.cc:304 + * #1 0x0039dc7e in pbxt_end at ha_pbxt.cc:947 + * #2 0x0039dd27 in pbxt_panic at ha_pbxt.cc:1289 + * #3 0x001d619e in ha_finalize_handlerton at handler.cc:391 + * #4 0x00279d22 in plugin_deinitialize at sql_plugin.cc:816 + * #5 0x0027bcf5 in reap_plugins at sql_plugin.cc:904 + * #6 0x0027c38c in plugin_thdvar_cleanup at sql_plugin.cc:2513 + * #7 0x000c0db2 in THD::~THD at sql_class.cc:934 + * #8 0x003b025b in myxt_destroy_thread at myxt_xt.cc:2999 + * #9 0x003b66b5 in xn_xres_run_recovery_thread at restart_xt.cc:3196 + * #10 0x003cbfbb in thr_main at thread_xt.cc:1020 + * myxt_destroy_thread(mysql_thread, TRUE); + */ + xres_recovery_thread = NULL; return NULL; } @@ -3204,6 +3260,23 @@ xtPublic void xt_xres_start_database_rec sprintf(name, "DB-RECOVERY-%s", xt_last_directory_of_path(mysql_real_data_home)); xt_remove_dir_char(name); - XTThreadPtr thread = xt_create_daemon(self, name); - xt_run_thread(self, thread, xn_xres_run_recovery_thread); + + xres_recovery_thread = xt_create_daemon(self, name); + xt_run_thread(self, xres_recovery_thread, xn_xres_run_recovery_thread); +} + +xtPublic void xt_xres_wait_for_recovery(XTThreadPtr self) +{ + XTThreadPtr thr_rec; + + /* {MYSQL-THREAD-KILL} + * Stack above shows that his is possible! + */ + if ((thr_rec = xres_recovery_thread) && (self != xres_recovery_thread)) { + xtThreadID tid = thr_rec->t_id; + + xt_terminate_thread(self, thr_rec); + + xt_wait_for_thread(tid, TRUE); + } } === modified file 'storage/pbxt/src/restart_xt.h' --- a/storage/pbxt/src/restart_xt.h 2009-08-17 11:12:36 +0000 +++ b/storage/pbxt/src/restart_xt.h 2009-09-03 06:15:03 +0000 @@ -131,6 +131,7 @@ xtWord8 xt_bytes_since_last_checkpoint(s void xt_print_log_record(xtLogID log, off_t offset, XTXactLogBufferDPtr record); void xt_dump_xlogs(struct XTDatabase *db, xtLogID start_log); -xtPublic void xt_xres_start_database_recovery(XTThreadPtr self, const char *path); +void xt_xres_start_database_recovery(XTThreadPtr self); +void xt_xres_wait_for_recovery(XTThreadPtr self); #endif === modified file 'storage/pbxt/src/strutil_xt.cc' --- a/storage/pbxt/src/strutil_xt.cc 2009-08-18 07:46:53 +0000 +++ b/storage/pbxt/src/strutil_xt.cc 2009-09-03 06:15:03 +0000 @@ -368,7 +368,7 @@ xtPublic void xt_int8_to_byte_size(xtInt /* Version number must also be set in configure.in! */ xtPublic c_char *xt_get_version(void) { - return "1.0.08c RC"; + return "1.0.08d RC"; } /* Copy and URL decode! */ === modified file 'storage/pbxt/src/tabcache_xt.cc' --- a/storage/pbxt/src/tabcache_xt.cc 2009-08-17 11:12:36 +0000 +++ b/storage/pbxt/src/tabcache_xt.cc 2009-09-03 06:15:03 +0000 @@ -1178,7 +1178,10 @@ static void *tabc_fr_run_thread(XTThread } } + /* + * {MYSQL-THREAD-KILL} myxt_destroy_thread(mysql_thread, TRUE); + */ return NULL; } === modified file 'storage/pbxt/src/thread_xt.cc' --- a/storage/pbxt/src/thread_xt.cc 2009-08-17 11:12:36 +0000 +++ b/storage/pbxt/src/thread_xt.cc 2009-09-03 06:15:03 +0000 @@ -625,7 +625,7 @@ static c_char *thr_get_err_string(int xt case XT_ERR_NO_REFERENCED_ROW: str = "Constraint: `%s`"; break; // "Foreign key '%s', referenced row does not exist" case XT_ERR_ROW_IS_REFERENCED: str = "Constraint: `%s`"; break; // "Foreign key '%s', has a referencing row" case XT_ERR_BAD_DICTIONARY: str = "Internal dictionary does not match MySQL dictionary"; break; - case XT_ERR_LOADING_MYSQL_DIC: str = "Error %s loading MySQL .frm file"; break; + case XT_ERR_LOADING_MYSQL_DIC: str = "Error loading %s.frm file, MySQL error: %s"; break; case XT_ERR_COLUMN_IS_NOT_NULL: str = "Column `%s` is NOT NULL"; break; case XT_ERR_INCORRECT_NO_OF_COLS: str = "Incorrect number of columns near %s"; break; case XT_ERR_FK_ON_TEMP_TABLE: str = "Cannot create foreign key on temporary table"; break; @@ -656,6 +656,8 @@ static c_char *thr_get_err_string(int xt case XT_ERR_NEW_TYPE_OF_XLOG: str = "Transaction log %s, is using a newer format, upgrade required"; break; case XT_ERR_NO_BEFORE_IMAGE: str = "Internal error: no before image"; break; case XT_ERR_FK_REF_TEMP_TABLE: str = "Foreign key may not reference temporary table"; break; + case XT_ERR_MYSQL_SHUTDOWN: str = "Cannot open table, MySQL has shutdown"; break; + case XT_ERR_MYSQL_NO_THREAD: str = "Cannot create thread, MySQL has shutdown"; break; default: str = "Unknown XT error"; break; } return str; @@ -1026,6 +1028,11 @@ extern "C" void *thr_main(void *data) outer_(); xt_free_thread(self); + + /* {MYSQL-THREAD-KILL} + * Clean up any remaining MySQL thread! + */ + myxt_delete_remaining_thread(); return return_data; } === modified file 'storage/pbxt/src/xaction_xt.cc' --- a/storage/pbxt/src/xaction_xt.cc 2009-08-17 11:12:36 +0000 +++ b/storage/pbxt/src/xaction_xt.cc 2009-09-03 06:15:03 +0000 @@ -2529,7 +2529,8 @@ static void *xn_sw_run_thread(XTThreadPt int count; void *mysql_thread; - mysql_thread = myxt_create_thread(); + if (!(mysql_thread = myxt_create_thread())) + xt_throw(self); while (!self->t_quit) { try_(a) { @@ -2586,7 +2587,10 @@ static void *xn_sw_run_thread(XTThreadPt db->db_sw_idle = XT_THREAD_BUSY; } + /* + * {MYSQL-THREAD-KILL} myxt_destroy_thread(mysql_thread, TRUE); + */ return NULL; } === modified file 'storage/pbxt/src/xactlog_xt.cc' --- a/storage/pbxt/src/xactlog_xt.cc 2009-08-18 07:46:53 +0000 +++ b/storage/pbxt/src/xactlog_xt.cc 2009-09-03 06:15:03 +0000 @@ -2610,7 +2610,10 @@ static void *xlog_wr_run_thread(XTThread db->db_wr_idle = XT_THREAD_BUSY; } + /* + * {MYSQL-THREAD-KILL} myxt_destroy_thread(mysql_thread, TRUE); + */ return NULL; } === modified file 'storage/pbxt/src/xt_errno.h' --- a/storage/pbxt/src/xt_errno.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/xt_errno.h 2009-09-03 06:15:03 +0000 @@ -117,6 +117,8 @@ #define XT_ERR_NEW_TYPE_OF_XLOG -93 #define XT_ERR_NO_BEFORE_IMAGE -94 #define XT_ERR_FK_REF_TEMP_TABLE -95 +#define XT_ERR_MYSQL_SHUTDOWN -98 +#define XT_ERR_MYSQL_NO_THREAD -99 #ifdef XT_WIN #define XT_ENOMEM ERROR_NOT_ENOUGH_MEMORY === modified file 'storage/xtradb/fil/fil0fil.c' --- a/storage/xtradb/fil/fil0fil.c 2009-09-01 11:59:54 +0000 +++ b/storage/xtradb/fil/fil0fil.c 2009-09-03 13:20:22 +0000 @@ -45,7 +45,9 @@ Created 10/25/1995 Heikki Tuuri #include "trx0trx.h" #include "trx0sys.h" #include "pars0pars.h" +#include "row0row.h" #include "row0mysql.h" +#include "que0que.h" /* @@ -3137,7 +3139,7 @@ skip_info: rec_offs_init(offsets_); - fprintf(stderr, "InnoDB: Progress in %:"); + fprintf(stderr, "%s", "InnoDB: Progress in %:"); for (offset = 0; offset < size_bytes; offset += UNIV_PAGE_SIZE) { success = os_file_read(file, page, === modified file 'storage/xtradb/mtr/mtr0mtr.c' --- a/storage/xtradb/mtr/mtr0mtr.c 2009-06-25 01:43:25 +0000 +++ b/storage/xtradb/mtr/mtr0mtr.c 2009-09-03 13:20:22 +0000 @@ -32,6 +32,7 @@ Created 11/26/1995 Heikki Tuuri #include "page0types.h" #include "mtr0log.h" #include "log0log.h" +#include "buf0flu.h" /********************************************************************* Releases the item in the slot given. */ === modified file 'storage/xtradb/srv/srv0srv.c' --- a/storage/xtradb/srv/srv0srv.c 2009-09-01 11:59:54 +0000 +++ b/storage/xtradb/srv/srv0srv.c 2009-09-03 13:20:22 +0000 @@ -1815,7 +1815,6 @@ srv_printf_innodb_monitor( ulint btr_search_sys_subtotal; ulint lock_sys_subtotal; ulint recv_sys_subtotal; - ulint io_counter_subtotal; ulint i; trx_t* trx; === modified file 'storage/xtradb/srv/srv0start.c' --- a/storage/xtradb/srv/srv0start.c 2009-08-03 20:09:53 +0000 +++ b/storage/xtradb/srv/srv0start.c 2009-09-03 13:20:22 +0000 @@ -122,20 +122,6 @@ static char* srv_monitor_file_name; #define SRV_MAX_N_PENDING_SYNC_IOS 100 -/* Avoid warnings when using purify */ - -#ifdef HAVE_valgrind -static int inno_bcmp(register const char *s1, register const char *s2, - register uint len) -{ - while ((len-- != 0) && (*s1++ == *s2++)) - ; - - return(len + 1); -} -#define memcmp(A,B,C) inno_bcmp((A),(B),(C)) -#endif - static char* srv_parse_megabytes( === modified file 'strings/decimal.c' --- a/strings/decimal.c 2009-05-06 12:03:24 +0000 +++ b/strings/decimal.c 2009-09-03 13:20:22 +0000 @@ -306,7 +306,7 @@ int decimal_actual_fraction(decimal_t *f { for (i= DIG_PER_DEC1 - ((frac - 1) % DIG_PER_DEC1); *buf0 % powers10[i++] == 0; - frac--); + frac--) {} } return frac; } @@ -500,7 +500,7 @@ static void digits_bounds(decimal_t *fro stop= (int) ((buf_end - from->buf + 1) * DIG_PER_DEC1); i= 1; } - for (; *buf_end % powers10[i++] == 0; stop--); + for (; *buf_end % powers10[i++] == 0; stop--) {} *end_result= stop; /* index of position after last decimal digit (from 0) */ } @@ -1011,7 +1011,7 @@ static int ull2dec(ulonglong from, decim sanity(to); - for (intg1=1; from >= DIG_BASE; intg1++, from/=DIG_BASE); + for (intg1=1; from >= DIG_BASE; intg1++, from/=DIG_BASE) {} if (unlikely(intg1 > to->len)) { intg1=to->len; === added file 'support-files/ccfilter' --- a/support-files/ccfilter 1970-01-01 00:00:00 +0000 +++ b/support-files/ccfilter 2009-09-03 13:20:22 +0000 @@ -0,0 +1,104 @@ +#! /usr/bin/perl + +# Post-processor for compiler output to filter out warnings matched in +# support-files/compiler_warnings.supp. This makes it easier to check +# that no new warnings are introduced without needing to submit a build +# for Buildbot. +# +# Use by setting CC="ccfilter gcc" CXX="ccfilter gcc" before ./configure. +# +# By default, just filters the output for suppressed warnings. If the +# FAILONWARNING environment variable is set, then instead will fail the +# compile on encountering a non-suppressed warnings. + +use strict; +use warnings; + +my $suppressions; + +open STDOUT_COPY, ">&STDOUT" + or die "Failed to dup stdout: $!]n"; + +my $pid= open(PIPE, '-|'); + +if (!defined($pid)) { + die "Error: Cannot fork(): $!\n"; +} elsif (!$pid) { + # Child. + # actually want to send the STDERR to the parent, not the STDOUT. + # So shuffle things around a bit. + open STDERR, ">&STDOUT" + or die "Child: Failed to dup pipe to parent: $!\n"; + open STDOUT, ">&STDOUT_COPY" + or die "Child: Failed to dup parent stdout: $!\n"; + close STDOUT_COPY; + exec { $ARGV[0] } @ARGV; + die "Child: exec() failed: $!\n"; +} else { + # Parent. + close STDOUT_COPY; + my $cwd= qx(pwd); + chomp($cwd); + while (<PIPE>) { + my $line= $_; + if (/^(.*?):([0-9]+): [Ww]arning: (.*)$/) { + my ($file, $lineno, $msg)= ($1, $2, $3); + $file= "$cwd/$file"; + + next + if check_if_suppressed($file, $lineno, $msg); + die "$line\nGot warning, terminating.\n" + if $ENV{FAILONWARNING}; + print STDERR $line; + next; + } + + print STDERR $line; + } + close(PIPE); +} + +exit 0; + +sub check_if_suppressed { + my ($file, $lineno, $msg)= @_; + load_suppressions() unless defined($suppressions); + for my $s (@$suppressions) { + my ($file_re, $msg_re, $start, $end)= @$s; + if ($file =~ /$file_re/ && + $msg =~ /$msg_re/ && + (!defined($start) || $start <= $lineno) && + (!defined($end) || $end >= $lineno)) { + return 1; + } + } + return undef; +} + +sub load_suppressions { + # First find the suppressions file, might be we need to move up to + # the base directory. + my $path = "support-files/compiler_warnings.supp"; + my $exists; + for (1..10) { + $exists= -f $path; + last if $exists; + $path= '../'. $path; + } + die "Error: Could not find suppression file (out of source dir?).\n" + unless $exists; + + $suppressions= []; + open "F", "<", $path + or die "Error: Could not read suppression file '$path': $!\n"; + while (<F>) { + # Skip comment and empty lines. + next if /^\s*(\#.*)?$/; + die "Invalid syntax in suppression file '$path', line $.:\n$_" + unless /^\s*(.+?)\s*:\s*(.+?)\s*(?:[:]\s*([0-9]+)(?:-([0-9]+))?\s*)?$/; + my ($file_re, $line_re, $start, $end)= ($1, $2, $3, $4); + $end = $start + if defined($start) && !defined($end); + push @$suppressions, [$file_re, $line_re, $start, $end]; + } +} === modified file 'support-files/compiler_warnings.supp' --- a/support-files/compiler_warnings.supp 2008-01-11 17:39:43 +0000 +++ b/support-files/compiler_warnings.supp 2009-09-03 13:20:22 +0000 @@ -25,6 +25,9 @@ sql_yacc.cc : .*switch statement contain pars0grm.tab.c: .*'yyerrorlab' : unreferenced label.* _flex_tmp.c: .*not enough actual parameters for macro 'yywrap'.* pars0lex.l: .*conversion from 'ulint' to 'int', possible loss of data.* +btr/btr0cur\.c: .*value computed is not used.*: 3175-3375 +include/buf0buf\.ic: unused parameter ‘mtr’ +fil/fil0fil\.c: comparison between signed and unsigned : 3100-3199 # # bdb is not critical to keep up to date @@ -41,6 +44,12 @@ db_vrfy.c : .*comparison is always false .*/cmd-line-utils/readline/.* : .* # +# Ignore some warnings in libevent, which is not maintained by us. +# +.*/extra/libevent/.* : .*unused parameter.* +.*/extra/libevent/select\.c : .*comparison between signed and unsigned.* : 270-280 + +# # Ignore all conversion warnings on windows 64 # (Is safe as we are not yet supporting strings >= 2G) # @@ -75,6 +84,17 @@ db_vrfy.c : .*comparison is always false storage/maria/ma_pagecache.c: .*'info_check_pin' defined but not used # +# I think these are due to mix of C and C++. +# +storage/pbxt/ : typedef.*was ignored in this declaration + + +# +# Groff warnings on OpenSUSE. +# +.*/dbug/.*(groff|<standard input>) : .* + +# # Unexplanable (?) stuff # listener.cc : .*conversion from 'SOCKET' to 'int'.*

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2727)
by knielsen＠knielsen-hq.org 04 Sep '09

04 Sep '09

#At lp:maria 2727 knielsen(a)knielsen-hq.org 2009-09-04 PBXT fixes. - Silence a compiler warning. - Don't build pbms object files now that we disabled pbms. modified: storage/pbxt/src/Makefile.am storage/pbxt/src/datalog_xt.cc per-file messages: storage/pbxt/src/Makefile.am We disabled PBMS, so don't build it. storage/pbxt/src/datalog_xt.cc Silence a warning about signed/unsigned comparison. === modified file 'storage/pbxt/src/Makefile.am' --- a/storage/pbxt/src/Makefile.am 2009-08-31 11:07:44 +0000 +++ b/storage/pbxt/src/Makefile.am 2009-09-04 07:29:34 +0000 @@ -30,7 +30,7 @@ libpbxt_la_SOURCES = bsearch_xt.cc cache datadic_xt.cc datalog_xt.cc filesys_xt.cc hashtab_xt.cc \ ha_pbxt.cc heap_xt.cc index_xt.cc linklist_xt.cc \ memory_xt.cc myxt_xt.cc pthread_xt.cc restart_xt.cc \ - pbms_enabled.cc sortedlist_xt.cc strutil_xt.cc \ + sortedlist_xt.cc strutil_xt.cc \ tabcache_xt.cc table_xt.cc trace_xt.cc thread_xt.cc \ systab_xt.cc ha_xtsys.cc discover_xt.cc \ util_xt.cc xaction_xt.cc xactlog_xt.cc lock_xt.cc locklist_xt.cc @@ -49,4 +49,4 @@ libpbxt_a_SOURCES = $(libpbxt_la_SOURCE libpbxt_a_CXXFLAGS = $(AM_CXXFLAGS) libpbxt_a_CFLAGS = $(AM_CFLAGS) -std=c99 -EXTRA_DIST = CMakeLists.txt +EXTRA_DIST = CMakeLists.txt pbms_enabled.cc === modified file 'storage/pbxt/src/datalog_xt.cc' --- a/storage/pbxt/src/datalog_xt.cc 2009-09-03 06:15:03 +0000 +++ b/storage/pbxt/src/datalog_xt.cc 2009-09-04 07:29:34 +0000 @@ -410,7 +410,7 @@ static void dl_recover_log(XTThreadPtr s ASSERT_NS(seq_read.sl_log_eof == seq_read.sl_rec_log_offset); data_log->dlf_log_eof = seq_read.sl_rec_log_offset; - if (data_log->dlf_log_eof < sizeof(XTXactLogHeaderDRec)) { + if ((size_t) data_log->dlf_log_eof < sizeof(XTXactLogHeaderDRec)) { data_log->dlf_log_eof = sizeof(XTXactLogHeaderDRec); if (!dl_create_log_header(data_log, seq_read.sl_log_file, self)) xt_throw(self);

1 0

[Maria-developers] Progress (by Sanja): Avoid flushing keycache to disk in case of DROPping table (53)
by worklog-noreply＠askmonty.org 03 Sep '09

03 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Avoid flushing keycache to disk in case of DROPping table CREATION DATE..: Thu, 27 Aug 2009, 11:12 SUPERVISOR.....: Monty IMPLEMENTOR....: Sanja COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 53 (http://askmonty.org/worklog/?tid=53) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 13 ESTIMATE.......: 11 (hours remain) ORIG. ESTIMATE.: 24 PROGRESS NOTES: -=-=(Sanja - Thu, 03 Sep 2009, 19:43)=-=- New patch and its testing Worked 8 hours and estimate 11 hours remain (original estimate unchanged). -=-=(Sanja - Tue, 01 Sep 2009, 15:20)=-=- moving to 5.0 part1 Worked 2 hours and estimate 19 hours remain (original estimate unchanged). -=-=(Sanja - Tue, 01 Sep 2009, 15:19)=-=- first patch Worked 3 hours and estimate 21 hours remain (original estimate unchanged). -=-=(Sanja - Thu, 27 Aug 2009, 11:15)=-=- High-Level Specification modified. --- /tmp/wklog.53.old.23490 2009-08-27 11:15:24.000000000 +0300 +++ /tmp/wklog.53.new.23490 2009-08-27 11:15:24.000000000 +0300 @@ -1 +1,3 @@ - +To make special handler call 'prepare to drop' which mark table as changed (in +case of server crash just before removing table it force table to be repaired) +then flush keycache without writing on disk. DESCRIPTION: When DROP TABLE executed keycache flushed to disk before removing table which is wasting of resources. So better to have table removed from keycache without writing changes on disk. HIGH-LEVEL SPECIFICATION: To make special handler call 'prepare to drop' which mark table as changed (in case of server crash just before removing table it force table to be repaired) then flush keycache without writing on disk. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Progress (by Sanja): Avoid flushing keycache to disk in case of DROPping table (53)
by worklog-noreply＠askmonty.org 03 Sep '09

03 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Avoid flushing keycache to disk in case of DROPping table CREATION DATE..: Thu, 27 Aug 2009, 11:12 SUPERVISOR.....: Monty IMPLEMENTOR....: Sanja COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 53 (http://askmonty.org/worklog/?tid=53) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 13 ESTIMATE.......: 11 (hours remain) ORIG. ESTIMATE.: 24 PROGRESS NOTES: -=-=(Sanja - Thu, 03 Sep 2009, 19:43)=-=- New patch and its testing Worked 8 hours and estimate 11 hours remain (original estimate unchanged). -=-=(Sanja - Tue, 01 Sep 2009, 15:20)=-=- moving to 5.0 part1 Worked 2 hours and estimate 19 hours remain (original estimate unchanged). -=-=(Sanja - Tue, 01 Sep 2009, 15:19)=-=- first patch Worked 3 hours and estimate 21 hours remain (original estimate unchanged). -=-=(Sanja - Thu, 27 Aug 2009, 11:15)=-=- High-Level Specification modified. --- /tmp/wklog.53.old.23490 2009-08-27 11:15:24.000000000 +0300 +++ /tmp/wklog.53.new.23490 2009-08-27 11:15:24.000000000 +0300 @@ -1 +1,3 @@ - +To make special handler call 'prepare to drop' which mark table as changed (in +case of server crash just before removing table it force table to be repaired) +then flush keycache without writing on disk. DESCRIPTION: When DROP TABLE executed keycache flushed to disk before removing table which is wasting of resources. So better to have table removed from keycache without writing changes on disk. HIGH-LEVEL SPECIFICATION: To make special handler call 'prepare to drop' which mark table as changed (in case of server crash just before removing table it force table to be repaired) then flush keycache without writing on disk. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Rev 2720: Fix for in file:///Users/hakan/work/monty_program/maria/
by Hakan Kuecuekyilmaz 03 Sep '09

03 Sep '09

At file:///Users/hakan/work/monty_program/maria/ ------------------------------------------------------------ revno: 2720 revision-id: hakan(a)askmonty.org-20090903163555-hds72helhcy5htqq parent: knielsen(a)knielsen-hq.org-20090805072137-wg97dcem1cxnzt3p committer: Hakan Kuecuekyilmaz <hakan(a)askmonty.org> branch nick: maria timestamp: Thu 2009-09-03 18:35:55 +0200 message: Fix for mysqlslap: setting --engine does not get replicated http://bugs.mysql.com/bug.php?id=46967 Problem was that a "set storage_engine = <storage_engine>" is not replicated. Fixed by adding an explicit Engine = <storage_engine> to the build_table_string() function. Open question: Should MySQL's replication actually replicate the set storage_engine command or not? Answer: no it should not (documented). === modified file 'client/mysqlslap.c' --- a/client/mysqlslap.c 2009-04-25 10:05:32 +0000 +++ b/client/mysqlslap.c 2009-09-03 16:35:55 +0000 @@ -880,6 +880,13 @@ } dynstr_append(&table_string, ")"); + + if (default_engine) + { + dynstr_append(&table_string, " Engine = "); + dynstr_append(&table_string, default_engine); + } + ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL|MY_FAE|MY_WME)); ptr->string = (char *)my_malloc(table_string.length+1,

1 0

Re: [Maria-developers] MariaDB and the SCA
by Kristian Nielsen 03 Sep '09

03 Sep '09

Kristian Nielsen <knielsen(a)knielsen-hq.org> writes: > Davi Arnaut <Davi.Arnaut(a)Sun.COM> writes: > >> I've been monitoring the maria-developers list and there are some >> patches that we would like to gradually incorporate into our >> branches. For a practical example, i would like to merge into 5.1 >> parts of this (http://tinyurl.com/q2ulgt) patch by Kristian. > Negotiations between Monty Program and Sun have started some time ago on a > deal which would cover how Sun will get the rights it needs (ie. SCA) to > include the MariaDB code into the MySQL code, and how Monty Program will > receive compensation for giving these rights. As I understand it, these > negotiations are however still in the early phase. Apparently, this is moving nowhere :-(. Davi, I'm sorry there has been no progress on this. We discussed this again in MariaDB, and would like to get things working better for now. What Monty suggested was that it would work for Sun if we submit specific patches under a BSD-new license. I realise that the http://tinyurl.com/q2ulgt patch request is now so old that it may be of no interest to you any longer. But if you have a request for this or other patches, please send them. All developers at Monty Program have the ability to submit their work to Sun at their own discretion, so we should be able to make this work on a low-overhead technician-to-technician level. - Kristian.

6 7

[Maria-developers] Rev 2721: Fix for coding style. in file:///Users/hakan/work/monty_program/maria/
by Hakan Kuecuekyilmaz 03 Sep '09

03 Sep '09

At file:///Users/hakan/work/monty_program/maria/ ------------------------------------------------------------ revno: 2721 revision-id: hakan(a)askmonty.org-20090903150639-y5s9y2ylowfesyr4 parent: hakan(a)askmonty.org-20090902214617-5aljfm0c229hzdpu committer: Hakan Kuecuekyilmaz <hakan(a)askmonty.org> branch nick: maria timestamp: Thu 2009-09-03 17:06:39 +0200 message: Fix for coding style. === modified file 'client/mysqlslap.c' --- a/client/mysqlslap.c 2009-09-02 21:46:17 +0000 +++ b/client/mysqlslap.c 2009-09-03 15:06:39 +0000 @@ -881,7 +881,8 @@ dynstr_append(&table_string, ")"); - if (default_engine) { + if (default_engine) + { dynstr_append(&table_string, " Engine = "); dynstr_append(&table_string, default_engine); }

1 0

[Maria-developers] [Branch ~maria-captains/maria/5.1] Rev 2726: Updated documentation files to reflect MariaDB and not the Maria storage engine or MySQL
by noreply＠launchpad.net 03 Sep '09

03 Sep '09

Merge authors: Michael Widenius (monty) ------------------------------------------------------------ revno: 2726 [merge] committer: Michael Widenius <monty(a)mysql.com> branch nick: maria timestamp: Thu 2009-09-03 17:56:46 +0300 message: Updated documentation files to reflect MariaDB and not the Maria storage engine or MySQL Added (rewritten) patch from Percona to get extended statistics in slow.log: - Added handling of 'set' variables to set_var.cc. Changed sql_mode to use this - Added extra logging to slow log of 'Thread_id, Schema, Query Cache hit, Rows sent and Rows examined' - Added optional logging to slow log, through log_slow_verbosity, of query plan statistics - Added new user variables log_slow_rate_limit, log_slow_verbosity, log_slow_filter - Added log-slow-file as synonym for 'slow-log-file', as most slow-log variables starts with 'log-slow' - Added log-slow-time as synonym for long-query-time Some trivial MyISAM optimizations: - In prepare for drop, flush key blocks - Don't call mi_lock_database if my_disable_locking is used ****** Automatic merge with trunc ****** Updated documentation files to reflect MariaDB and not the Maria storage engine or MySQL Added (rewritten) patch from Percona to get extended statistics in slow.log: - Added handling of 'set' variables to set_var.cc. Changed sql_mode to use this - Added extra logging to slow log of 'Thread_id, Schema, Query Cache hit, Rows sent and Rows examined' - Added optional logging to slow log, through log_slow_verbosity, of query plan statistics - Added new user variables log_slow_rate_limit, log_slow_verbosity, log_slow_filter - Added log-slow-file as synonym for 'slow-log-file', as most slow-log variables starts with 'log-slow' - Added log-slow-time as synonym for long-query-time Some trivial MyISAM optimizations: - In prepare for drop, flush key blocks - Don't call mi_lock_database if my_disable_locking is used added: mysql-test/r/log_slow.result mysql-test/t/log_slow.test sql/log_slow.h modified: KNOWN_BUGS.txt README mysql-test/r/variables.result sql/Makefile.am sql/event_data_objects.cc sql/events.cc sql/filesort.cc sql/log.cc sql/mysql_priv.h sql/mysqld.cc sql/set_var.cc sql/set_var.h sql/slave.cc sql/sp_head.cc sql/sql_cache.cc sql/sql_class.cc sql/sql_class.h sql/sql_parse.cc sql/sql_select.cc sql/sql_show.cc sql/strfunc.cc storage/myisam/mi_extra.c storage/myisam/mi_locking.c -- lp:maria https://code.launchpad.net/~maria-captains/maria/5.1 Your team Maria developers is subscribed to branch lp:maria. To unsubscribe from this branch go to https://code.launchpad.net/~maria-captains/maria/5.1/+edit-subscription.

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2726)
by knielsen＠knielsen-hq.org 03 Sep '09

03 Sep '09

#At lp:maria 2726 knielsen(a)knielsen-hq.org 2009-09-03 Fix most Compiler warnings seen in buildbot. Add suppressions for a few warnings that cannot be meaningfully fixed by MariaDB developers. Changes for XtraDB, PBXT, and YaSSL also submitted upstream. Also add a `ccfilter` wrapper that can be used to filter out suppressed warnings in a local build (to check that new warnings are not introduced). added: support-files/ccfilter modified: client/mysqlbinlog.cc config/ac-macros/misc.m4 extra/yassl/include/yassl_int.hpp extra/yassl/src/handshake.cpp extra/yassl/src/yassl_imp.cpp extra/yassl/src/yassl_int.cpp extra/yassl/taocrypt/include/modes.hpp extra/yassl/taocrypt/src/asn.cpp mysys/my_compress.c sql/mysqld.cc sql/strfunc.cc storage/pbxt/src/discover_xt.cc storage/xtradb/fil/fil0fil.c storage/xtradb/mtr/mtr0mtr.c storage/xtradb/srv/srv0srv.c storage/xtradb/srv/srv0start.c strings/decimal.c support-files/compiler_warnings.supp per-file messages: client/mysqlbinlog.cc Fix compiler warnings. config/ac-macros/misc.m4 Fix wrong naming, autoconfig requires _cv_ in cached names. extra/yassl/include/yassl_int.hpp Fix compiler warnings. extra/yassl/src/handshake.cpp Fix compiler warnings. extra/yassl/src/yassl_imp.cpp Fix compiler warnings. extra/yassl/src/yassl_int.cpp Fix compiler warnings. extra/yassl/taocrypt/include/modes.hpp Fix compiler warnings. extra/yassl/taocrypt/src/asn.cpp Fix compiler warnings. mysys/my_compress.c Fix compiler warnings. sql/mysqld.cc Fix compiler warnings. sql/strfunc.cc Fix compiler warnings. storage/pbxt/src/discover_xt.cc Fix compiler warnings. storage/xtradb/fil/fil0fil.c Fix compiler warnings. storage/xtradb/mtr/mtr0mtr.c Fix compiler warnings. storage/xtradb/srv/srv0srv.c Fix compiler warnings. storage/xtradb/srv/srv0start.c Fix compiler warnings. strings/decimal.c Fix compiler warnings. support-files/ccfilter Add helper for suppressing compiler warnings in local developer source tree. Allows to check for not introducing new warnings into Buildbot without having to actually run the build through Buildbot. support-files/compiler_warnings.supp Suppress a few warnings that cannot be meaningfully fixed in source code. === modified file 'client/mysqlbinlog.cc' --- a/client/mysqlbinlog.cc 2009-02-13 16:41:47 +0000 +++ b/client/mysqlbinlog.cc 2009-09-03 13:20:22 +0000 @@ -689,8 +689,8 @@ Exit_status process_event(PRINT_EVENT_IN Format events are not concerned by --offset and such, we always need to read them to be able to process the wanted events. */ - if ((rec_count >= offset) && - ((my_time_t)(ev->when) >= start_datetime) || + if (((rec_count >= offset) && + ((my_time_t)(ev->when) >= start_datetime)) || (ev_type == FORMAT_DESCRIPTION_EVENT)) { if (ev_type != FORMAT_DESCRIPTION_EVENT) === modified file 'config/ac-macros/misc.m4' --- a/config/ac-macros/misc.m4 2008-02-26 17:38:43 +0000 +++ b/config/ac-macros/misc.m4 2009-09-03 13:20:22 +0000 @@ -594,15 +594,15 @@ dnl ------------------------------------ dnl MYSQL_NEEDS_MYSYS_NEW AC_DEFUN([MYSQL_NEEDS_MYSYS_NEW], -[AC_CACHE_CHECK([needs mysys_new helpers], mysql_use_mysys_new, +[AC_CACHE_CHECK([needs mysys_new helpers], mysql_cv_use_mysys_new, [ AC_LANG_PUSH(C++) AC_TRY_LINK([], [ class A { public: int b; }; A *a=new A; a->b=10; delete a; -], mysql_use_mysys_new=no, mysql_use_mysys_new=yes) +], mysql_cv_use_mysys_new=no, mysql_cv_use_mysys_new=yes) AC_LANG_POP(C++) ]) -if test "$mysql_use_mysys_new" = "yes" +if test "$mysql_cv_use_mysys_new" = "yes" then AC_DEFINE([USE_MYSYS_NEW], [1], [Needs to use mysys_new helpers]) fi === modified file 'extra/yassl/include/yassl_int.hpp' --- a/extra/yassl/include/yassl_int.hpp 2008-11-18 16:45:44 +0000 +++ b/extra/yassl/include/yassl_int.hpp 2009-09-03 13:20:22 +0000 @@ -441,7 +441,7 @@ public: const Ciphers& GetCiphers() const; const DH_Parms& GetDH_Parms() const; const Stats& GetStats() const; - const VerifyCallback getVerifyCallback() const; + VerifyCallback getVerifyCallback() const; pem_password_cb GetPasswordCb() const; void* GetUserData() const; bool GetSessionCacheOff() const; === modified file 'extra/yassl/src/handshake.cpp' --- a/extra/yassl/src/handshake.cpp 2008-04-28 16:24:05 +0000 +++ b/extra/yassl/src/handshake.cpp 2009-09-03 13:20:22 +0000 @@ -789,7 +789,7 @@ void processReply(SSL& ssl) { if (ssl.GetError()) return; - if (DoProcessReply(ssl)) + if (DoProcessReply(ssl)) { // didn't complete process if (!ssl.getSocket().IsNonBlocking()) { // keep trying now, blocking ok @@ -799,6 +799,7 @@ void processReply(SSL& ssl) else // user will have try again later, non blocking ssl.SetError(YasslError(SSL_ERROR_WANT_READ)); + } } @@ -872,11 +873,12 @@ void sendServerKeyExchange(SSL& ssl, Buf // send change cipher void sendChangeCipher(SSL& ssl, BufferOutput buffer) { - if (ssl.getSecurity().get_parms().entity_ == server_end) + if (ssl.getSecurity().get_parms().entity_ == server_end) { if (ssl.getSecurity().get_resuming()) ssl.verifyState(clientKeyExchangeComplete); else ssl.verifyState(clientFinishedComplete); + } if (ssl.GetError()) return; ChangeCipherSpec ccs; === modified file 'extra/yassl/src/yassl_imp.cpp' --- a/extra/yassl/src/yassl_imp.cpp 2009-02-13 16:41:47 +0000 +++ b/extra/yassl/src/yassl_imp.cpp 2009-09-03 13:20:22 +0000 @@ -1304,7 +1304,7 @@ void ServerHello::Process(input_buffer&, else ssl.useSecurity().use_connection().sessionID_Set_ = false; - if (ssl.getSecurity().get_resuming()) + if (ssl.getSecurity().get_resuming()) { if (memcmp(session_id_, ssl.getSecurity().get_resume().GetID(), ID_LEN) == 0) { ssl.set_masterSecret(ssl.getSecurity().get_resume().GetSecret()); @@ -1319,6 +1319,7 @@ void ServerHello::Process(input_buffer&, ssl.useSecurity().set_resuming(false); ssl.useLog().Trace("server denied resumption"); } + } if (ssl.CompressionOn() && !compression_method_) ssl.UnSetCompression(); // server isn't supporting yaSSL zlib request === modified file 'extra/yassl/src/yassl_int.cpp' --- a/extra/yassl/src/yassl_int.cpp 2009-02-13 16:41:47 +0000 +++ b/extra/yassl/src/yassl_int.cpp 2009-09-03 13:20:22 +0000 @@ -1833,7 +1833,7 @@ SSL_CTX::GetCA_List() const } -const VerifyCallback SSL_CTX::getVerifyCallback() const +VerifyCallback SSL_CTX::getVerifyCallback() const { return verifyCallback_; } === modified file 'extra/yassl/taocrypt/include/modes.hpp' --- a/extra/yassl/taocrypt/include/modes.hpp 2007-03-23 12:43:09 +0000 +++ b/extra/yassl/taocrypt/include/modes.hpp 2009-09-03 13:20:22 +0000 @@ -95,11 +95,12 @@ inline void Mode_BASE::Process(byte* out { if (mode_ == ECB) ECB_Process(out, in, sz); - else if (mode_ == CBC) + else if (mode_ == CBC) { if (dir_ == ENCRYPTION) CBC_Encrypt(out, in, sz); else CBC_Decrypt(out, in, sz); + } } === modified file 'extra/yassl/taocrypt/src/asn.cpp' --- a/extra/yassl/taocrypt/src/asn.cpp 2009-02-13 16:41:47 +0000 +++ b/extra/yassl/taocrypt/src/asn.cpp 2009-09-03 13:20:22 +0000 @@ -780,11 +780,12 @@ void CertDecoder::GetDate(DateType dt) memcpy(date, source_.get_current(), length); source_.advance(length); - if (!ValidateDate(date, b, dt) && verify_) + if (!ValidateDate(date, b, dt) && verify_) { if (dt == BEFORE) source_.SetError(BEFORE_DATE_E); else source_.SetError(AFTER_DATE_E); + } // save for later use if (dt == BEFORE) { @@ -1061,7 +1062,7 @@ word32 DecodeDSA_Signature(byte* decoded return 0; } word32 rLen = GetLength(source); - if (rLen != 20) + if (rLen != 20) { if (rLen == 21) { // zero at front, eat source.next(); --rLen; @@ -1074,6 +1075,7 @@ word32 DecodeDSA_Signature(byte* decoded source.SetError(DSA_SZ_E); return 0; } + } memcpy(decoded, source.get_buffer() + source.get_index(), rLen); source.advance(rLen); @@ -1083,7 +1085,7 @@ word32 DecodeDSA_Signature(byte* decoded return 0; } word32 sLen = GetLength(source); - if (sLen != 20) + if (sLen != 20) { if (sLen == 21) { source.next(); // zero at front, eat --sLen; @@ -1096,6 +1098,7 @@ word32 DecodeDSA_Signature(byte* decoded source.SetError(DSA_SZ_E); return 0; } + } memcpy(decoded + rLen, source.get_buffer() + source.get_index(), sLen); source.advance(sLen); === modified file 'mysys/my_compress.c' --- a/mysys/my_compress.c 2009-05-22 12:38:50 +0000 +++ b/mysys/my_compress.c 2009-09-03 13:20:22 +0000 @@ -81,12 +81,13 @@ my_bool my_compress(uchar *packet, size_ This fix is safe, since such memory is only used internally by zlib, so we will not hide any bugs in mysql this way. */ -void *my_az_allocator(void *dummy, unsigned int items, unsigned int size) +void *my_az_allocator(void *dummy __attribute__((unused)), unsigned int items, + unsigned int size) { return my_malloc((size_t)items*(size_t)size, IF_VALGRIND(MY_ZEROFILL, MYF(0))); } -void my_az_free(void *dummy, void *address) +void my_az_free(void *dummy __attribute__((unused)), void *address) { my_free(address, MYF(MY_ALLOW_ZERO_PTR)); } === modified file 'sql/mysqld.cc' --- a/sql/mysqld.cc 2009-05-19 09:28:05 +0000 +++ b/sql/mysqld.cc 2009-09-03 13:20:22 +0000 @@ -1002,6 +1002,7 @@ static void close_connections(void) } +#ifdef HAVE_CLOSE_SERVER_SOCK static void close_socket(my_socket sock, const char *info) { DBUG_ENTER("close_socket"); @@ -1021,6 +1022,7 @@ static void close_socket(my_socket sock, } DBUG_VOID_RETURN; } +#endif static void close_server_sock() === modified file 'sql/strfunc.cc' --- a/sql/strfunc.cc 2009-04-25 10:05:32 +0000 +++ b/sql/strfunc.cc 2009-09-03 13:20:22 +0000 @@ -148,7 +148,7 @@ static uint parse_name(TYPELIB *lib, con } } else - for (; pos != end && *pos != '=' && *pos !=',' ; pos++); + for (; pos != end && *pos != '=' && *pos !=',' ; pos++) {} uint var_len= (uint) (pos - start); /* Determine which flag it is */ === modified file 'storage/pbxt/src/discover_xt.cc' --- a/storage/pbxt/src/discover_xt.cc 2009-08-17 11:12:36 +0000 +++ b/storage/pbxt/src/discover_xt.cc 2009-09-03 13:20:22 +0000 @@ -493,8 +493,8 @@ mysql_prepare_create_table(THD *thd, HA_ } /* Don't pack rows in old tables if the user has requested this */ if ((sql_field->flags & BLOB_FLAG) || - sql_field->sql_type == MYSQL_TYPE_VARCHAR && - create_info->row_type != ROW_TYPE_FIXED) + (sql_field->sql_type == MYSQL_TYPE_VARCHAR && + create_info->row_type != ROW_TYPE_FIXED)) (*db_options)|= HA_OPTION_PACK_RECORD; it2.rewind(); } @@ -963,7 +963,7 @@ mysql_prepare_create_table(THD *thd, HA_ sql_field->sql_type == MYSQL_TYPE_VARCHAR || sql_field->pack_flag & FIELDFLAG_BLOB))) { - if (column_nr == 0 && (sql_field->pack_flag & FIELDFLAG_BLOB) || + if ((column_nr == 0 && (sql_field->pack_flag & FIELDFLAG_BLOB)) || sql_field->sql_type == MYSQL_TYPE_VARCHAR) key_info->flags|= HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY; else === modified file 'storage/xtradb/fil/fil0fil.c' --- a/storage/xtradb/fil/fil0fil.c 2009-09-01 11:59:54 +0000 +++ b/storage/xtradb/fil/fil0fil.c 2009-09-03 13:20:22 +0000 @@ -45,7 +45,9 @@ Created 10/25/1995 Heikki Tuuri #include "trx0trx.h" #include "trx0sys.h" #include "pars0pars.h" +#include "row0row.h" #include "row0mysql.h" +#include "que0que.h" /* @@ -3137,7 +3139,7 @@ skip_info: rec_offs_init(offsets_); - fprintf(stderr, "InnoDB: Progress in %:"); + fprintf(stderr, "%s", "InnoDB: Progress in %:"); for (offset = 0; offset < size_bytes; offset += UNIV_PAGE_SIZE) { success = os_file_read(file, page, === modified file 'storage/xtradb/mtr/mtr0mtr.c' --- a/storage/xtradb/mtr/mtr0mtr.c 2009-06-25 01:43:25 +0000 +++ b/storage/xtradb/mtr/mtr0mtr.c 2009-09-03 13:20:22 +0000 @@ -32,6 +32,7 @@ Created 11/26/1995 Heikki Tuuri #include "page0types.h" #include "mtr0log.h" #include "log0log.h" +#include "buf0flu.h" /********************************************************************* Releases the item in the slot given. */ === modified file 'storage/xtradb/srv/srv0srv.c' --- a/storage/xtradb/srv/srv0srv.c 2009-09-01 11:59:54 +0000 +++ b/storage/xtradb/srv/srv0srv.c 2009-09-03 13:20:22 +0000 @@ -1815,7 +1815,6 @@ srv_printf_innodb_monitor( ulint btr_search_sys_subtotal; ulint lock_sys_subtotal; ulint recv_sys_subtotal; - ulint io_counter_subtotal; ulint i; trx_t* trx; === modified file 'storage/xtradb/srv/srv0start.c' --- a/storage/xtradb/srv/srv0start.c 2009-08-03 20:09:53 +0000 +++ b/storage/xtradb/srv/srv0start.c 2009-09-03 13:20:22 +0000 @@ -122,20 +122,6 @@ static char* srv_monitor_file_name; #define SRV_MAX_N_PENDING_SYNC_IOS 100 -/* Avoid warnings when using purify */ - -#ifdef HAVE_valgrind -static int inno_bcmp(register const char *s1, register const char *s2, - register uint len) -{ - while ((len-- != 0) && (*s1++ == *s2++)) - ; - - return(len + 1); -} -#define memcmp(A,B,C) inno_bcmp((A),(B),(C)) -#endif - static char* srv_parse_megabytes( === modified file 'strings/decimal.c' --- a/strings/decimal.c 2009-05-06 12:03:24 +0000 +++ b/strings/decimal.c 2009-09-03 13:20:22 +0000 @@ -306,7 +306,7 @@ int decimal_actual_fraction(decimal_t *f { for (i= DIG_PER_DEC1 - ((frac - 1) % DIG_PER_DEC1); *buf0 % powers10[i++] == 0; - frac--); + frac--) {} } return frac; } @@ -500,7 +500,7 @@ static void digits_bounds(decimal_t *fro stop= (int) ((buf_end - from->buf + 1) * DIG_PER_DEC1); i= 1; } - for (; *buf_end % powers10[i++] == 0; stop--); + for (; *buf_end % powers10[i++] == 0; stop--) {} *end_result= stop; /* index of position after last decimal digit (from 0) */ } @@ -1011,7 +1011,7 @@ static int ull2dec(ulonglong from, decim sanity(to); - for (intg1=1; from >= DIG_BASE; intg1++, from/=DIG_BASE); + for (intg1=1; from >= DIG_BASE; intg1++, from/=DIG_BASE) {} if (unlikely(intg1 > to->len)) { intg1=to->len; === added file 'support-files/ccfilter' --- a/support-files/ccfilter 1970-01-01 00:00:00 +0000 +++ b/support-files/ccfilter 2009-09-03 13:20:22 +0000 @@ -0,0 +1,104 @@ +#! /usr/bin/perl + +# Post-processor for compiler output to filter out warnings matched in +# support-files/compiler_warnings.supp. This makes it easier to check +# that no new warnings are introduced without needing to submit a build +# for Buildbot. +# +# Use by setting CC="ccfilter gcc" CXX="ccfilter gcc" before ./configure. +# +# By default, just filters the output for suppressed warnings. If the +# FAILONWARNING environment variable is set, then instead will fail the +# compile on encountering a non-suppressed warnings. + +use strict; +use warnings; + +my $suppressions; + +open STDOUT_COPY, ">&STDOUT" + or die "Failed to dup stdout: $!]n"; + +my $pid= open(PIPE, '-|'); + +if (!defined($pid)) { + die "Error: Cannot fork(): $!\n"; +} elsif (!$pid) { + # Child. + # actually want to send the STDERR to the parent, not the STDOUT. + # So shuffle things around a bit. + open STDERR, ">&STDOUT" + or die "Child: Failed to dup pipe to parent: $!\n"; + open STDOUT, ">&STDOUT_COPY" + or die "Child: Failed to dup parent stdout: $!\n"; + close STDOUT_COPY; + exec { $ARGV[0] } @ARGV; + die "Child: exec() failed: $!\n"; +} else { + # Parent. + close STDOUT_COPY; + my $cwd= qx(pwd); + chomp($cwd); + while (<PIPE>) { + my $line= $_; + if (/^(.*?):([0-9]+): [Ww]arning: (.*)$/) { + my ($file, $lineno, $msg)= ($1, $2, $3); + $file= "$cwd/$file"; + + next + if check_if_suppressed($file, $lineno, $msg); + die "$line\nGot warning, terminating.\n" + if $ENV{FAILONWARNING}; + print STDERR $line; + next; + } + + print STDERR $line; + } + close(PIPE); +} + +exit 0; + +sub check_if_suppressed { + my ($file, $lineno, $msg)= @_; + load_suppressions() unless defined($suppressions); + for my $s (@$suppressions) { + my ($file_re, $msg_re, $start, $end)= @$s; + if ($file =~ /$file_re/ && + $msg =~ /$msg_re/ && + (!defined($start) || $start <= $lineno) && + (!defined($end) || $end >= $lineno)) { + return 1; + } + } + return undef; +} + +sub load_suppressions { + # First find the suppressions file, might be we need to move up to + # the base directory. + my $path = "support-files/compiler_warnings.supp"; + my $exists; + for (1..10) { + $exists= -f $path; + last if $exists; + $path= '../'. $path; + } + die "Error: Could not find suppression file (out of source dir?).\n" + unless $exists; + + $suppressions= []; + open "F", "<", $path + or die "Error: Could not read suppression file '$path': $!\n"; + while (<F>) { + # Skip comment and empty lines. + next if /^\s*(\#.*)?$/; + die "Invalid syntax in suppression file '$path', line $.:\n$_" + unless /^\s*(.+?)\s*:\s*(.+?)\s*(?:[:]\s*([0-9]+)(?:-([0-9]+))?\s*)?$/; + my ($file_re, $line_re, $start, $end)= ($1, $2, $3, $4); + $end = $start + if defined($start) && !defined($end); + push @$suppressions, [$file_re, $line_re, $start, $end]; + } +} === modified file 'support-files/compiler_warnings.supp' --- a/support-files/compiler_warnings.supp 2008-01-11 17:39:43 +0000 +++ b/support-files/compiler_warnings.supp 2009-09-03 13:20:22 +0000 @@ -25,6 +25,9 @@ sql_yacc.cc : .*switch statement contain pars0grm.tab.c: .*'yyerrorlab' : unreferenced label.* _flex_tmp.c: .*not enough actual parameters for macro 'yywrap'.* pars0lex.l: .*conversion from 'ulint' to 'int', possible loss of data.* +btr/btr0cur\.c: .*value computed is not used.*: 3175-3375 +include/buf0buf\.ic: unused parameter ‘mtr’ +fil/fil0fil\.c: comparison between signed and unsigned : 3100-3199 # # bdb is not critical to keep up to date @@ -41,6 +44,12 @@ db_vrfy.c : .*comparison is always false .*/cmd-line-utils/readline/.* : .* # +# Ignore some warnings in libevent, which is not maintained by us. +# +.*/extra/libevent/.* : .*unused parameter.* +.*/extra/libevent/select\.c : .*comparison between signed and unsigned.* : 270-280 + +# # Ignore all conversion warnings on windows 64 # (Is safe as we are not yet supporting strings >= 2G) # @@ -75,6 +84,17 @@ db_vrfy.c : .*comparison is always false storage/maria/ma_pagecache.c: .*'info_check_pin' defined but not used # +# I think these are due to mix of C and C++. +# +storage/pbxt/ : typedef.*was ignored in this declaration + + +# +# Groff warnings on OpenSUSE. +# +.*/dbug/.*(groff|<standard input>) : .* + +# # Unexplanable (?) stuff # listener.cc : .*conversion from 'SOCKET' to 'int'.*

1 0

[Maria-developers] Rev 2720: Fix for in file:///Users/hakan/work/monty_program/maria/
by Hakan Kuecuekyilmaz 03 Sep '09

03 Sep '09

At file:///Users/hakan/work/monty_program/maria/ ------------------------------------------------------------ revno: 2720 revision-id: hakan(a)askmonty.org-20090902214617-5aljfm0c229hzdpu parent: knielsen(a)knielsen-hq.org-20090805072137-wg97dcem1cxnzt3p committer: Hakan Kuecuekyilmaz <hakan(a)askmonty.org> branch nick: maria timestamp: Wed 2009-09-02 23:46:17 +0200 message: Fix for mysqlslap: setting --engine does not get replicated http://bugs.mysql.com/bug.php?id=46967 Problem was that a "set storage_engine = <storage_engine>" is not replicated. Fixed by adding an explicit Engine = <storage_engine> to the build_table_string() function. Open question: Should MySQL's replication actually replicate the set storage_engine command or not? Answer: no it should not (documented). === modified file 'client/mysqlslap.c' --- a/client/mysqlslap.c 2009-04-25 10:05:32 +0000 +++ b/client/mysqlslap.c 2009-09-02 21:46:17 +0000 @@ -880,6 +880,12 @@ } dynstr_append(&table_string, ")"); + + if (default_engine) { + dynstr_append(&table_string, " Engine = "); + dynstr_append(&table_string, default_engine); + } + ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL|MY_FAE|MY_WME)); ptr->string = (char *)my_malloc(table_string.length+1,

2 1

[Maria-developers] Rev 2791: Preventing keycache flush to disk before DROP/TRUNCATE TABLE for myisam (for review) in file:///Users/bell/maria/bzr/work-mysql-keycache-5.0/
by sanja＠askmonty.org 02 Sep '09

02 Sep '09

At file:///Users/bell/maria/bzr/work-mysql-keycache-5.0/ ------------------------------------------------------------ revno: 2791 revision-id: sanja(a)askmonty.org-20090902145945-0rif23n1nth7naj7 parent: sanja(a)askmonty.org-20090828183603-gvbl4vmd8buph91m committer: sanja(a)askmonty.org branch nick: work-mysql-keycache-5.0 timestamp: Wed 2009-09-02 17:59:45 +0300 message: Preventing keycache flush to disk before DROP/TRUNCATE TABLE for myisam (for review) === modified file 'include/my_base.h' --- a/include/my_base.h 2009-07-17 08:43:53 +0000 +++ b/include/my_base.h 2009-09-02 14:59:45 +0000 @@ -173,7 +173,9 @@ Inform handler that an "INSERT...ON DUPLICATE KEY UPDATE" will be executed. This condition is unset by HA_EXTRA_NO_IGNORE_DUP_KEY. */ - HA_EXTRA_INSERT_WITH_UPDATE + HA_EXTRA_INSERT_WITH_UPDATE, + /* We are going to drop the table */ + HA_EXTRA_PREPARE_FOR_DROP }; /* The following is parameter to ha_panic() */ === modified file 'myisam/mi_close.c' --- a/myisam/mi_close.c 2009-03-20 09:18:14 +0000 +++ b/myisam/mi_close.c 2009-09-02 14:59:45 +0000 @@ -63,8 +63,9 @@ { if (share->kfile >= 0 && flush_key_blocks(share->key_cache, share->kfile, - share->temporary ? FLUSH_IGNORE_CHANGED : - FLUSH_RELEASE)) + (share->temporary || share->deleting) ? + FLUSH_IGNORE_CHANGED : + FLUSH_RELEASE)) error=my_errno; if (share->kfile >= 0) { === modified file 'myisam/mi_extra.c' --- a/myisam/mi_extra.c 2007-07-27 09:30:25 +0000 +++ b/myisam/mi_extra.c 2009-09-02 14:59:45 +0000 @@ -276,9 +276,9 @@ pthread_mutex_unlock(&THR_LOCK_myisam); break; case HA_EXTRA_PREPARE_FOR_DELETE: + DBUG_PRINT("info", ("HA_EXTRA_PREPARE_FOR_DELETE")); pthread_mutex_lock(&THR_LOCK_myisam); share->last_version= 0L; /* Impossible version */ -#ifdef __WIN__REMOVE_OBSOLETE_WORKAROUND /* Close the isam and data files as Win32 can't drop an open table */ pthread_mutex_lock(&share->intern_lock); if (flush_key_blocks(share->key_cache, share->kfile, @@ -295,6 +295,8 @@ info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); error=end_io_cache(&info->rec_cache); } + pthread_mutex_unlock(&share->intern_lock); +#ifdef __WIN__REMOVE_OBSOLETE_WORKAROUND if (info->lock_type != F_UNLCK && ! info->was_locked) { info->was_locked=info->lock_type; @@ -322,7 +324,6 @@ } } share->kfile= -1; /* Files aren't open anymore */ - pthread_mutex_unlock(&share->intern_lock); #endif pthread_mutex_unlock(&THR_LOCK_myisam); break; @@ -366,6 +367,11 @@ case HA_EXTRA_CHANGE_KEY_TO_DUP: mi_extra_keyflag(info, function); break; + case HA_EXTRA_PREPARE_FOR_DROP: + /* Signals about intent to delete this table */ + share->deleting= TRUE; + _mi_mark_file_changed(info); + break; case HA_EXTRA_KEY_CACHE: case HA_EXTRA_NO_KEY_CACHE: default: === modified file 'myisam/mi_locking.c' --- a/myisam/mi_locking.c 2007-01-03 07:52:50 +0000 +++ b/myisam/mi_locking.c 2009-09-02 14:59:45 +0000 @@ -547,7 +547,7 @@ { uint old_lock=info->lock_type; share->global_changed=0; - lock_error=mi_lock_database(info,F_WRLCK); + lock_error= my_disable_locking ? 0 : mi_lock_database(info,F_WRLCK); /* Its not fatal even if we couldn't get the lock ! */ if (share->state.open_count > 0) { @@ -557,7 +557,7 @@ sizeof(share->state.header), MYF(MY_NABP)); } - if (!lock_error) + if (!lock_error && !my_disable_locking) lock_error=mi_lock_database(info,old_lock); } return test(lock_error || write_error); === modified file 'myisam/myisamdef.h' --- a/myisam/myisamdef.h 2009-01-15 10:48:31 +0000 +++ b/myisam/myisamdef.h 2009-09-02 14:59:45 +0000 @@ -203,6 +203,7 @@ not_flushed, temporary,delay_key_write, concurrent_insert; + my_bool deleting; /* we are going to delete this table */ #ifdef THREAD THR_LOCK lock; pthread_mutex_t intern_lock; /* Locking for use with _locking */ === modified file 'sql/lock.cc' --- a/sql/lock.cc 2009-04-14 17:20:13 +0000 +++ b/sql/lock.cc 2009-09-02 14:59:45 +0000 @@ -907,10 +907,12 @@ DBUG_RETURN(-1); table_list->table= table; + table->s->deleting= table_list->deleting; /* Return 1 if table is in use */ DBUG_RETURN(test(remove_table_from_cache(thd, db, table_list->table_name, - RTFC_NO_FLAG))); + RTFC_NO_FLAG, + table_list->deleting))); } === modified file 'sql/mysql_priv.h' --- a/sql/mysql_priv.h 2009-05-27 10:20:57 +0000 +++ b/sql/mysql_priv.h 2009-09-02 14:59:45 +0000 @@ -1187,7 +1187,7 @@ #define RTFC_WAIT_OTHER_THREAD_FLAG 0x0002 #define RTFC_CHECK_KILLED_FLAG 0x0004 bool remove_table_from_cache(THD *thd, const char *db, const char *table, - uint flags); + uint flags, my_bool deleting); bool close_cached_tables(THD *thd, bool wait_for_refresh, TABLE_LIST *tables); void copy_field_from_tmp_record(Field *field,int offset); === modified file 'sql/sql_base.cc' --- a/sql/sql_base.cc 2009-07-16 12:19:22 +0000 +++ b/sql/sql_base.cc 2009-09-02 14:59:45 +0000 @@ -342,7 +342,7 @@ for (TABLE_LIST *table= tables; table; table= table->next_local) { if (remove_table_from_cache(thd, table->db, table->table_name, - RTFC_OWNED_BY_THD_FLAG)) + RTFC_OWNED_BY_THD_FLAG, table->deleting)) found=1; } if (!found) @@ -6099,7 +6099,7 @@ */ bool remove_table_from_cache(THD *thd, const char *db, const char *table_name, - uint flags) + uint flags, my_bool deleting) { char key[MAX_DBKEY_LENGTH]; uint key_length; @@ -6166,7 +6166,10 @@ result= result || (flags & RTFC_OWNED_BY_THD_FLAG); } while (unused_tables && !unused_tables->s->version) + { + unused_tables->s->deleting= deleting; VOID(hash_delete(&open_cache,(byte*) unused_tables)); + } if (result && (flags & RTFC_WAIT_OTHER_THREAD_FLAG)) { /* === modified file 'sql/sql_delete.cc' --- a/sql/sql_delete.cc 2009-07-13 15:11:16 +0000 +++ b/sql/sql_delete.cc 2009-09-02 14:59:45 +0000 @@ -920,6 +920,7 @@ HA_CREATE_INFO create_info; char path[FN_REFLEN]; TABLE **table_ptr; + TABLE_LIST *table; bool error; DBUG_ENTER("mysql_truncate"); @@ -967,6 +968,10 @@ } if (!ha_check_storage_engine_flag(table_type, HTON_CAN_RECREATE)) goto trunc_by_del; + + for (table= table_list; table; table= table->next_local) + table->deleting= TRUE; /* to trigger HA_PREPARE_FOR_DROP */ + if (lock_and_wait_for_table_name(thd, table_list)) DBUG_RETURN(TRUE); } === modified file 'sql/sql_table.cc' --- a/sql/sql_table.cc 2009-06-29 13:17:01 +0000 +++ b/sql/sql_table.cc 2009-09-02 14:59:45 +0000 @@ -228,6 +228,9 @@ LINT_INIT(alias); + for (table= tables; table; table= table->next_local) + table->deleting= TRUE; + if (!drop_temporary && lock_table_names(thd, tables)) DBUG_RETURN(1); @@ -252,7 +255,7 @@ abort_locked_tables(thd, db, table->table_name); remove_table_from_cache(thd, db, table->table_name, RTFC_WAIT_OTHER_THREAD_FLAG | - RTFC_CHECK_KILLED_FLAG); + RTFC_CHECK_KILLED_FLAG, FALSE); drop_locked_tables(thd, db, table->table_name); if (thd->killed) { @@ -1966,7 +1969,8 @@ /* Wait until all there are no other threads that has this table open */ remove_table_from_cache(thd, table->s->db, - table->s->table_name, RTFC_WAIT_OTHER_THREAD_FLAG); + table->s->table_name, + RTFC_WAIT_OTHER_THREAD_FLAG, FALSE); DBUG_VOID_RETURN; } @@ -2365,7 +2369,7 @@ remove_table_from_cache(thd, table->table->s->db, table->table->s->table_name, RTFC_WAIT_OTHER_THREAD_FLAG | - RTFC_CHECK_KILLED_FLAG); + RTFC_CHECK_KILLED_FLAG, FALSE); thd->exit_cond(old_message); if (thd->killed) goto err; @@ -2586,7 +2590,8 @@ { pthread_mutex_lock(&LOCK_open); remove_table_from_cache(thd, table->table->s->db, - table->table->s->table_name, RTFC_NO_FLAG); + table->table->s->table_name, RTFC_NO_FLAG, + FALSE); pthread_mutex_unlock(&LOCK_open); } /* May be something modified consequently we have to invalidate cache */ @@ -3996,7 +4001,7 @@ { VOID(table->file->extra(HA_EXTRA_FORCE_REOPEN)); // Use new file /* Mark in-use copies old */ - remove_table_from_cache(thd,db,table_name,RTFC_NO_FLAG); + remove_table_from_cache(thd, db, table_name, RTFC_NO_FLAG, FALSE); /* end threads waiting on lock */ mysql_lock_abort(thd,table); } === modified file 'sql/table.cc' --- a/sql/table.cc 2009-06-17 13:54:01 +0000 +++ b/sql/table.cc 2009-09-02 14:59:45 +0000 @@ -993,7 +993,12 @@ int error=0; DBUG_ENTER("closefrm"); if (table->db_stat) - error=table->file->close(); + { + if (table->s->deleting) + table->file->extra(HA_EXTRA_PREPARE_FOR_DROP); + error= table->file->close(); + } + my_free((char*) table->alias, MYF(MY_ALLOW_ZERO_PTR)); table->alias= 0; if (table->field) === modified file 'sql/table.h' --- a/sql/table.h 2009-06-17 13:54:01 +0000 +++ b/sql/table.h 2009-09-02 14:59:45 +0000 @@ -202,6 +202,7 @@ locking of this table for writing. FALSE - otherwise. */ my_bool system_table; + bool deleting; /* going to delete this table */ } TABLE_SHARE; @@ -713,6 +714,7 @@ ... SELECT implementation). */ bool create; + bool deleting; /* going to delete this table */ /** Indicates what triggers we need to pre-load for this TABLE_LIST

1 0

[Maria-developers] Rev 2724: Ability to stop mysql-test-run.pl temporary (useful for them who run buildbot slave on thier work computers) in file:///Users/bell/maria/bzr/work-maria-5.1/
by sanja＠askmonty.org 02 Sep '09

02 Sep '09

At file:///Users/bell/maria/bzr/work-maria-5.1/ ------------------------------------------------------------ revno: 2724 revision-id: sanja(a)askmonty.org-20090902091710-4p0hjwa0usl3e8av parent: psergey(a)askmonty.org-20090901115954-nge2brzh3akkmtlz committer: sanja(a)askmonty.org branch nick: work-maria-5.1 timestamp: Wed 2009-09-02 12:17:10 +0300 message: Ability to stop mysql-test-run.pl temporary (useful for them who run buildbot slave on thier work computers) === modified file 'mysql-test/lib/mtr_misc.pl' --- a/mysql-test/lib/mtr_misc.pl 2009-02-19 09:01:25 +0000 +++ b/mysql-test/lib/mtr_misc.pl 2009-09-02 09:17:10 +0000 @@ -174,4 +174,21 @@ select(undef, undef, undef, ($millis/1000)); } +sub mtr_wait_lock_file { + die "usage: mtr_wait_lock_file(path_to_file, show_stop)" unless @_ == 2; + my ($file, $show)= @_; + my $waited= 0; + + while ( -e $file) + { + if (!$waited && $show) + { + print "\nGOING TO STOP...\n"; + } + mtr_milli_sleep(1000); + $waited= 1; + } + return ($waited); +} + 1; === modified file 'mysql-test/mysql-test-run.pl' --- a/mysql-test/mysql-test-run.pl 2009-06-22 08:06:35 +0000 +++ b/mysql-test/mysql-test-run.pl 2009-09-02 09:17:10 +0000 @@ -252,6 +252,11 @@ my $opt_parallel= $ENV{MTR_PARALLEL} || 1; +# lock file to stop tests +my $opt_stop_file= $ENV{MTR_STOP_FILE}; +# print message when test suite goint to be stopped +my $opt_show_stop= 0; + select(STDOUT); $| = 1; # Automatically flush STDOUT @@ -450,6 +455,15 @@ my $s= IO::Select->new(); $s->add($server); while (1) { + if ($opt_stop_file) + { + if (mtr_wait_lock_file($opt_stop_file, $opt_show_stop)) + { + # We were waiting so restart timer process + $suite_timeout_proc->kill(); + $suite_timeout_proc= My::SafeProcess->timer(suite_timeout()); + } + } my @ready = $s->can_read(1); # Wake up once every second foreach my $sock (@ready) { if ($sock == $server) { @@ -931,6 +945,8 @@ 'warnings!' => \$opt_warnings, 'timestamp' => \&report_option, 'timediff' => \&report_option, + 'stop-file=s' => \$opt_stop_file, + 'show-stop' => \$opt_show_stop, 'help|h' => \$opt_usage, 'list-options' => \$opt_list_options, @@ -5392,6 +5408,12 @@ warnings Scan the log files for warnings. Use --nowarnings to turn off. + stop-file=file (also MTR_STOP_FILE environment variable) if this + file detected mysql test will not start new tests + until the file will be removed. + show-stop works with stop-file, print message when mysql test + switches to waiting to removing the file mode. + sleep=SECONDS Passed to mysqltest, will be used as fixed sleep time debug-sync-timeout=NUM Set default timeout for WAIT_FOR debug sync actions. Disable facility with NUM=0.

1 0

[Maria-developers] Issues with static linking
by Kristian Nielsen 02 Sep '09

02 Sep '09

The issue of static vs. dynamic linking keeps coming up in relation to building of MariaDB (and MySQL). I thought I would share some thoughts on this. There is a tradition for linking statically in MySQL for performance and other reasons. However, these days there are a number of problems with static linking, to the extent that I think static linking of system libraries is really impractical. Now, in a *fully* static binary that does not load any shared libraries at run-time, static linking works ok. However, in most real-life situations, there is always some amount of dynamic linking: - Nowadays glibc uses dynamic loading at runtime to handle NSS. And while this can be disabled, it is strongly discouraged: http://gnu.gds.tuwien.ac.at/software/libc/FAQ.html#s-2.22 - Support for dynamic loading of pluggable storage engines requires dynamic loading at runtime. - Even just installing a single UDF (user-defined function) into the server requires dynamic loading of a shared library. - Client libraries (libmysql, libmysqld) may be used in dynamic shared objects built by users (like the Amarok plugin using libmysqld). So a binary built for general distribution really has no choice but to support some dynamic linking (users building on their own can do differently, of course). The problem now is the following: Consider a binary B that loads a dynamic shared object S. Assume that both B and S use the same library L. Now if B links L statically, part of L, but not necessarily all of it, will be included inside B. These parts of L will be of some version v1 of L determined when B was built, and they will be visible to S. But S may use other parts of L not included in B, and these may be of a different version v2 of L, as determined by how S was linked and loaded. The result is that S will be using a mix of two different versions of L. This very rarely work. Typically, L will be eg. libc, and B will contain parts of libc from the build machine, while S will use the libc installed on the machine on which B is run. This is the basic problem. Because of the way linking works in Linux and similar systems, libraries used by dynamically loaded objects must themselves be linked dynamically to avoid this problem with mixed versions. This applies to things like libc and libz. On the other hand objects that are internally part of MariaDB (libmystrings, libmyisam, etc.) are no problem to link statically. There are also a number of bugs on MySQL relating to this. So it is not just an academic problem. The conclusion as far as I can see is that for MariaDB binaries, we should link system libraries dynamically, or things will just not work correctly for our users. Unless someone knows a way to avoid these problems. ----------------------------------------------------------------------- Just to recap, the main benefits of static linking are, to my knowledge: 1. Performance. On i386, position-independent code (as needed in shared objects) needs code like this in the prologue of every function that accesses global names (like calling other functions): call label label: pop %ebx This consumes cycles, and reduces available registers on an already register-starved architecture. 2. System independence. Upgrading system libraries on the host does not affect the application, eg. less change of breakage due to incompatibility (though this can be also a disadvantage in case of eg. security fixes). As far as I know, nowadays, these advantages are much smaller: 1. On x86_64, the CPU architecture has been fixed to support position-independent code with little or no extra overhead (like pc-relative addressing modes). 2. Glibc has become much better at preserving binary compatibility across versions. ----------------------------------------------------------------------- Hope this helps, - Kristian.

1 0

[Maria-developers] [Branch ~maria-captains/maria/5.1] Rev 2725: MWL#17: Table elimination
by noreply＠launchpad.net 02 Sep '09

02 Sep '09

Merge authors: Sergey Petrunia (sergefp) Sergey Petrunia (sergefp) ------------------------------------------------------------ revno: 2725 [merge] committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1 timestamp: Wed 2009-09-02 12:42:28 +0400 message: MWL#17: Table elimination Pre-push merge to main. This should give the complete WL entry diff. added: mysql-test/lib/process-purecov-annotations.pl mysql-test/r/table_elim.result mysql-test/t/table_elim.test sql-bench/test-table-elimination.sh sql/opt_table_elimination.cc modified: .bzrignore include/my_global.h libmysqld/Makefile.am mysql-test/lib/mtr_gcov.pl mysql-test/mysql-test-run.pl mysql-test/r/mysql-bug41486.result mysql-test/r/ps_11bugs.result mysql-test/r/select.result mysql-test/r/subselect.result mysql-test/r/union.result mysql-test/t/index_merge_myisam.test mysql-test/t/mysql-bug41486.test mysql-test/valgrind.supp sql/CMakeLists.txt sql/Makefile.am sql/item.cc sql/item.h sql/item_cmpfunc.cc sql/item_cmpfunc.h sql/item_subselect.cc sql/item_subselect.h sql/item_sum.cc sql/item_sum.h sql/mysql_priv.h sql/mysqld.cc sql/sql_bitmap.h sql/sql_lex.cc sql/sql_lex.h sql/sql_list.h sql/sql_select.cc sql/sql_select.h sql/table.h -- lp:maria https://code.launchpad.net/~maria-captains/maria/5.1 Your team Maria developers is subscribed to branch lp:maria. To unsubscribe from this branch go to https://code.launchpad.net/~maria-captains/maria/5.1/+edit-subscription.

1 0

[Maria-developers] [Branch ~maria-captains/maria/5.1] Rev 2724: More comments
by noreply＠launchpad.net 02 Sep '09

02 Sep '09

------------------------------------------------------------ revno: 2724 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1 timestamp: Wed 2009-09-02 12:40:18 +0400 message: More comments modified: sql/opt_range.h -- lp:maria https://code.launchpad.net/~maria-captains/maria/5.1 Your team Maria developers is subscribed to branch lp:maria. To unsubscribe from this branch go to https://code.launchpad.net/~maria-captains/maria/5.1/+edit-subscription.

1 0

[Maria-developers] Rev 2725: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1/
by Sergey Petrunya 02 Sep '09

02 Sep '09

At file:///home/psergey/dev/maria-5.1/ ------------------------------------------------------------ revno: 2725 revision-id: psergey(a)askmonty.org-20090902084228-3ttm7xalqglzgjvs parent: psergey(a)askmonty.org-20090902084018-lv3qet4nhooc4cnu parent: psergey(a)askmonty.org-20090901222415-c3142zs9nstn2af0 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1 timestamp: Wed 2009-09-02 12:42:28 +0400 message: MWL#17: Table elimination Pre-push merge to main. This should give the complete WL entry diff. added: mysql-test/lib/process-purecov-annotations.pl processpurecovannota-20090825145959-3tgucpo20ms8b5yb-1 mysql-test/r/table_elim.result table_elim.result-20090603125022-nge13y0ohk1g2tt2-1 mysql-test/t/table_elim.test table_elim.test-20090603125018-ka3vcfrm07bsldz8-1 sql-bench/test-table-elimination.sh testtableelimination-20090616194329-gai92muve732qknl-1 sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 modified: .bzrignore sp1f-ignore-20001018235455-q4gxfbritt5f42nwix354ufpsvrf5ebj include/my_global.h sp1f-my_global.h-20010915021246-4vawdgfw4vg3tuxq6mejt7lrchcnceha libmysqld/Makefile.am sp1f-makefile.am-20010411110351-26htpk3ynkyh7pkfvnshztqrxx3few4g mysql-test/lib/mtr_gcov.pl sp1f-mtr_gcov.pl-20041230152646-g7ukx2aojk6vlfbrurijutdkqshwljue mysql-test/mysql-test-run.pl sp1f-mysqltestrun.pl-20041230152716-xjnn5ndv4rr4by6ijmj5a4ysubxc7qh3 mysql-test/r/mysql-bug41486.result mysqlbug41486.result-20090323135900-fobg67a3yzg0b7e8-1 mysql-test/r/ps_11bugs.result sp1f-ps_11bugs.result-20041012140047-4pktjlfeq27q6bxqfdsbcszr5nybv6zz mysql-test/r/select.result sp1f-select.result-20010103001548-znkoalxem6wchsbxizfosjhpfmhfyxuk mysql-test/r/subselect.result sp1f-subselect.result-20020512204640-zgegcsgavnfd7t7eyrf7ibuqomsw7uzo mysql-test/r/union.result sp1f-unions_one.result-20010725122836-ofxtwraxeohz7whhrmfdz57sl4a5prmp mysql-test/t/index_merge_myisam.test sp1f-index_merge_myisam.t-20060816114353-3p3p5fylcfj754j3d3td6zs6shnc5nu3 mysql-test/t/mysql-bug41486.test mysqlbug41486.test-20090323135900-fobg67a3yzg0b7e8-2 mysql-test/valgrind.supp sp1f-valgrind.supp-20050406142216-yg7xhezklqhgqlc3inx36vbghodhbovy sql/CMakeLists.txt sp1f-cmakelists.txt-20060831175237-esoeu5kpdtwjvehkghwy6fzbleniq2wy sql/Makefile.am sp1f-makefile.am-19700101030959-xsjdiakci3nqcdd4xl4yomwdl5eo2f3q sql/item.cc sp1f-item.cc-19700101030959-u7hxqopwpfly4kf5ctlyk2dvrq4l3dhn sql/item.h sp1f-item.h-19700101030959-rrkb43htudd62batmoteashkebcwykpa sql/item_cmpfunc.cc sp1f-item_cmpfunc.cc-19700101030959-hrk7pi2n6qpwxauufnkizirsoucdcx2e sql/item_cmpfunc.h sp1f-item_cmpfunc.h-19700101030959-pcvbjplo4e4ng7ibynfhcd6pjyem57gr sql/item_subselect.cc sp1f-item_subselect.cc-20020512204640-qep43aqhsfrwkqmrobni6czc3fqj36oo sql/item_subselect.h sp1f-item_subselect.h-20020512204640-qdg77wil56cxyhtc2bjjdrppxq3wqgh3 sql/item_sum.cc sp1f-item_sum.cc-19700101030959-4woo23bi3am2t2zvsddqbpxk7xbttdkm sql/item_sum.h sp1f-item_sum.h-19700101030959-ecgohlekwm355wxl5fv4zzq3alalbwyl sql/mysql_priv.h sp1f-mysql_priv.h-19700101030959-4fl65tqpop5zfgxaxkqotu2fa2ree5ci sql/mysqld.cc sp1f-mysqld.cc-19700101030959-zpswdvekpvixxzxf7gdtofzel7nywtfj sql/sql_bitmap.h sp1f-sql_bitmap.h-20031024204444-g4eiad7vopzqxe2trxmt3fn3xsvnomvj sql/sql_lex.cc sp1f-sql_lex.cc-19700101030959-4pizwlu5rqkti27gcwsvxkawq6bc2kph sql/sql_lex.h sp1f-sql_lex.h-19700101030959-sgldb2sooc7twtw5q7pgjx7qzqiaa3sn sql/sql_list.h sp1f-sql_list.h-19700101030959-hyddr5nvvtrgbk7llgnah4i4bf6ugspn sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb sql/sql_select.h sp1f-sql_select.h-19700101030959-oqegfxr76xlgmrzd6qlevonoibfnwzoz sql/table.h sp1f-table.h-19700101030959-dv72bajftxj5fbdjuajquappanuv2ija ------------------------------------------------------------ revno: 2719.2.31 revision-id: psergey(a)askmonty.org-20090901222415-c3142zs9nstn2af0 parent: psergey(a)askmonty.org-20090901222000-st4iicaxopeex2s1 parent: psergey(a)askmonty.org-20090901214116-n5bzyqlhgl0lamno committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r12 timestamp: Wed 2009-09-02 02:24:15 +0400 message: Merge modified: include/my_global.h sp1f-my_global.h-20010915021246-4vawdgfw4vg3tuxq6mejt7lrchcnceha sql-bench/test-table-elimination.sh testtableelimination-20090616194329-gai92muve732qknl-1 sql/item.cc sp1f-item.cc-19700101030959-u7hxqopwpfly4kf5ctlyk2dvrq4l3dhn sql/item.h sp1f-item.h-19700101030959-rrkb43htudd62batmoteashkebcwykpa sql/item_cmpfunc.cc sp1f-item_cmpfunc.cc-19700101030959-hrk7pi2n6qpwxauufnkizirsoucdcx2e sql/item_subselect.cc sp1f-item_subselect.cc-20020512204640-qep43aqhsfrwkqmrobni6czc3fqj36oo sql/item_subselect.h sp1f-item_subselect.h-20020512204640-qdg77wil56cxyhtc2bjjdrppxq3wqgh3 sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 sql/sql_list.h sp1f-sql_list.h-19700101030959-hyddr5nvvtrgbk7llgnah4i4bf6ugspn sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb ------------------------------------------------------------ revno: 2719.3.2 revision-id: psergey(a)askmonty.org-20090901214116-n5bzyqlhgl0lamno parent: psergey(a)askmonty.org-20090831200209-7gbnkr7omjbgld1x committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Wed 2009-09-02 01:41:16 +0400 message: MWL#17: Table elimination - Address review feedback R4: better comments, formatting modified: sql-bench/test-table-elimination.sh testtableelimination-20090616194329-gai92muve732qknl-1 sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.3.1 revision-id: psergey(a)askmonty.org-20090831200209-7gbnkr7omjbgld1x parent: psergey(a)askmonty.org-20090826210140-y2pba3ucdwnvr956 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Tue 2009-09-01 00:02:09 +0400 message: MWL#17: Table-elimination - Addressing review feedback, generation 4. modified: include/my_global.h sp1f-my_global.h-20010915021246-4vawdgfw4vg3tuxq6mejt7lrchcnceha sql/item.cc sp1f-item.cc-19700101030959-u7hxqopwpfly4kf5ctlyk2dvrq4l3dhn sql/item.h sp1f-item.h-19700101030959-rrkb43htudd62batmoteashkebcwykpa sql/item_cmpfunc.cc sp1f-item_cmpfunc.cc-19700101030959-hrk7pi2n6qpwxauufnkizirsoucdcx2e sql/item_subselect.cc sp1f-item_subselect.cc-20020512204640-qep43aqhsfrwkqmrobni6czc3fqj36oo sql/item_subselect.h sp1f-item_subselect.h-20020512204640-qdg77wil56cxyhtc2bjjdrppxq3wqgh3 sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 sql/sql_list.h sp1f-sql_list.h-19700101030959-hyddr5nvvtrgbk7llgnah4i4bf6ugspn sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb ------------------------------------------------------------ revno: 2719.2.30 revision-id: psergey(a)askmonty.org-20090901222000-st4iicaxopeex2s1 parent: psergey(a)askmonty.org-20090826210140-y2pba3ucdwnvr956 parent: psergey(a)askmonty.org-20090901115954-nge2brzh3akkmtlz committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r12 timestamp: Wed 2009-09-02 02:20:00 +0400 message: Merge removed: storage/pbxt/src/streaming_xt.cc streaming_xt.cc-20090326121724-x683v32twzr3fi0y-57 storage/pbxt/src/streaming_xt.h streaming_xt.h-20090326121724-x683v32twzr3fi0y-58 added: mysql-test/suite/pbxt/t/lowercase_table_grant-master.opt lowercase_table_gran-20090828130852-3duj73io0g345hxj-1 mysql-test/suite/pbxt/t/lowercase_table_qcache-master.opt lowercase_table_qcac-20090828130852-3duj73io0g345hxj-2 mysql-test/suite/pbxt/t/lowercase_view-master.opt lowercase_viewmaster-20090828130852-3duj73io0g345hxj-3 mysql-test/suite/pbxt/t/udf-master.opt udfmaster.opt-20090828130852-3duj73io0g345hxj-4 storage/pbxt/src/pbms_enabled.cc pbms_enabled.cc-20090818074502-tybcb62hp1kdrk3l-1 storage/pbxt/src/pbms_enabled.h pbms_enabled.h-20090818074502-tybcb62hp1kdrk3l-2 modified: BUILD/Makefile.am sp1f-makefile.am-20020102192940-dza66ux2yxyklupzjz4q3km3hvye5rnj libmysql/CMakeLists.txt sp1f-cmakelists.txt-20060831175237-3bbprkzqdl3gopcnlt3evmaikau67bcq mysql-test/suite/pbxt/r/alter_table.result alter_table.result-20090402100035-4ilk9i91sh65vjcb-5 mysql-test/suite/pbxt/r/analyze.result analyze.result-20090402100035-4ilk9i91sh65vjcb-7 mysql-test/suite/pbxt/r/auto_increment.result auto_increment.resul-20090402100035-4ilk9i91sh65vjcb-9 mysql-test/suite/pbxt/r/delete.result delete.result-20090402100035-4ilk9i91sh65vjcb-31 mysql-test/suite/pbxt/r/distinct.result distinct.result-20090402100035-4ilk9i91sh65vjcb-34 mysql-test/suite/pbxt/r/func_group.result func_group.result-20090402100035-4ilk9i91sh65vjcb-50 mysql-test/suite/pbxt/r/func_math.result func_math.result-20090402100035-4ilk9i91sh65vjcb-55 mysql-test/suite/pbxt/r/func_str.result func_str.result-20090402100035-4ilk9i91sh65vjcb-61 mysql-test/suite/pbxt/r/grant.result grant.result-20090402100035-4ilk9i91sh65vjcb-65 mysql-test/suite/pbxt/r/group_min_max.result group_min_max.result-20090402100035-4ilk9i91sh65vjcb-69 mysql-test/suite/pbxt/r/join.result join.result-20090402100035-4ilk9i91sh65vjcb-79 mysql-test/suite/pbxt/r/join_nested.result join_nested.result-20090402100035-4ilk9i91sh65vjcb-81 mysql-test/suite/pbxt/r/key.result key.result-20090402100035-4ilk9i91sh65vjcb-83 mysql-test/suite/pbxt/r/key_cache.result key_cache.result-20090402100035-4ilk9i91sh65vjcb-84 mysql-test/suite/pbxt/r/key_diff.result key_diff.result-20090402100035-4ilk9i91sh65vjcb-85 mysql-test/suite/pbxt/r/lowercase_view.result lowercase_view.resul-20090402100035-4ilk9i91sh65vjcb-96 mysql-test/suite/pbxt/r/mysqlshow.result mysqlshow.result-20090402100035-4ilk9i91sh65vjcb-101 mysql-test/suite/pbxt/r/null.result null.result-20090402100035-4ilk9i91sh65vjcb-104 mysql-test/suite/pbxt/r/null_key.result null_key.result-20090402100035-4ilk9i91sh65vjcb-105 mysql-test/suite/pbxt/r/partition_pruning.result partition_pruning.re-20090402100035-4ilk9i91sh65vjcb-117 mysql-test/suite/pbxt/r/pbxt_bugs.result pbxt_bugs.result-20090402100035-4ilk9i91sh65vjcb-120 mysql-test/suite/pbxt/r/pbxt_ref_int.result pbxt_ref_int.result-20090402100035-4ilk9i91sh65vjcb-123 mysql-test/suite/pbxt/r/preload.result preload.result-20090402100035-4ilk9i91sh65vjcb-127 mysql-test/suite/pbxt/r/ps_1general.result ps_1general.result-20090402100035-4ilk9i91sh65vjcb-130 mysql-test/suite/pbxt/r/range.result range.result-20090402100035-4ilk9i91sh65vjcb-132 mysql-test/suite/pbxt/r/schema.result schema.result-20090402100035-4ilk9i91sh65vjcb-138 mysql-test/suite/pbxt/r/select.result select.result-20090402100035-4ilk9i91sh65vjcb-139 mysql-test/suite/pbxt/r/select_safe.result select_safe.result-20090402100035-4ilk9i91sh65vjcb-141 mysql-test/suite/pbxt/r/subselect.result subselect.result-20090402100035-4ilk9i91sh65vjcb-146 mysql-test/suite/pbxt/r/type_enum.result type_enum.result-20090402100035-4ilk9i91sh65vjcb-155 mysql-test/suite/pbxt/r/type_ranges.result type_ranges.result-20090402100035-4ilk9i91sh65vjcb-159 mysql-test/suite/pbxt/r/type_timestamp.result type_timestamp.resul-20090402100035-4ilk9i91sh65vjcb-162 mysql-test/suite/pbxt/r/union.result union.result-20090402100035-4ilk9i91sh65vjcb-166 mysql-test/suite/pbxt/r/view_grant.result view_grant.result-20090402100035-4ilk9i91sh65vjcb-169 mysql-test/suite/pbxt/t/auto_increment.test auto_increment.test-20090402100035-4ilk9i91sh65vjcb-176 mysql-test/suite/pbxt/t/delete.test delete.test-20090402100035-4ilk9i91sh65vjcb-198 mysql-test/suite/pbxt/t/join_nested.test join_nested.test-20090402100035-4ilk9i91sh65vjcb-248 mysql-test/suite/pbxt/t/null.test null.test-20090402100035-4ilk9i91sh65vjcb-271 mysql-test/suite/pbxt/t/pbxt_bugs.test pbxt_bugs.test-20090402100035-4ilk9i91sh65vjcb-287 mysql-test/suite/pbxt/t/rename.test rename.test-20090402100035-4ilk9i91sh65vjcb-301 mysql-test/suite/pbxt/t/schema.test schema.test-20090402100035-4ilk9i91sh65vjcb-305 mysql-test/suite/pbxt/t/type_enum.test type_enum.test-20090402100035-4ilk9i91sh65vjcb-322 mysql-test/suite/pbxt/t/union.test union.test-20090402100035-4ilk9i91sh65vjcb-333 sql/CMakeLists.txt sp1f-cmakelists.txt-20060831175237-esoeu5kpdtwjvehkghwy6fzbleniq2wy storage/pbxt/ChangeLog changelog-20090326121724-x683v32twzr3fi0y-3 storage/pbxt/src/Makefile.am makefile.am-20090326121724-x683v32twzr3fi0y-13 storage/pbxt/src/cache_xt.cc cache_xt.cc-20090326121724-x683v32twzr3fi0y-16 storage/pbxt/src/cache_xt.h cache_xt.h-20090326121724-x683v32twzr3fi0y-17 storage/pbxt/src/ccutils_xt.cc ccutils_xt.cc-20090326121724-x683v32twzr3fi0y-18 storage/pbxt/src/database_xt.cc database_xt.cc-20090326121724-x683v32twzr3fi0y-20 storage/pbxt/src/datadic_xt.cc datadic_xt.cc-20090326121724-x683v32twzr3fi0y-22 storage/pbxt/src/datadic_xt.h datadic_xt.h-20090326121724-x683v32twzr3fi0y-23 storage/pbxt/src/datalog_xt.cc datalog_xt.cc-20090326121724-x683v32twzr3fi0y-24 storage/pbxt/src/datalog_xt.h datalog_xt.h-20090326121724-x683v32twzr3fi0y-25 storage/pbxt/src/discover_xt.cc discover_xt.cc-20090326121724-x683v32twzr3fi0y-26 storage/pbxt/src/filesys_xt.cc filesys_xt.cc-20090326121724-x683v32twzr3fi0y-28 storage/pbxt/src/filesys_xt.h filesys_xt.h-20090326121724-x683v32twzr3fi0y-29 storage/pbxt/src/ha_pbxt.cc ha_pbxt.cc-20090326121724-x683v32twzr3fi0y-30 storage/pbxt/src/ha_pbxt.h ha_pbxt.h-20090326121724-x683v32twzr3fi0y-31 storage/pbxt/src/ha_xtsys.cc ha_xtsys.cc-20090326121724-x683v32twzr3fi0y-32 storage/pbxt/src/ha_xtsys.h ha_xtsys.h-20090326121724-x683v32twzr3fi0y-33 storage/pbxt/src/hashtab_xt.cc hashtab_xt.cc-20090326121724-x683v32twzr3fi0y-34 storage/pbxt/src/heap_xt.cc heap_xt.cc-20090326121724-x683v32twzr3fi0y-36 storage/pbxt/src/heap_xt.h heap_xt.h-20090326121724-x683v32twzr3fi0y-37 storage/pbxt/src/index_xt.cc index_xt.cc-20090326121724-x683v32twzr3fi0y-38 storage/pbxt/src/index_xt.h index_xt.h-20090326121724-x683v32twzr3fi0y-39 storage/pbxt/src/lock_xt.cc lock_xt.cc-20090326121724-x683v32twzr3fi0y-42 storage/pbxt/src/lock_xt.h lock_xt.h-20090326121724-x683v32twzr3fi0y-43 storage/pbxt/src/locklist_xt.cc locklist_xt.cc-20090326121724-x683v32twzr3fi0y-44 storage/pbxt/src/locklist_xt.h locklist_xt.h-20090326121724-x683v32twzr3fi0y-45 storage/pbxt/src/memory_xt.cc memory_xt.cc-20090326121724-x683v32twzr3fi0y-46 storage/pbxt/src/memory_xt.h memory_xt.h-20090326121724-x683v32twzr3fi0y-47 storage/pbxt/src/myxt_xt.cc myxt_xt.cc-20090326121724-x683v32twzr3fi0y-48 storage/pbxt/src/myxt_xt.h myxt_xt.h-20090326121724-x683v32twzr3fi0y-49 storage/pbxt/src/pbms.h pbms.h-20090326121724-x683v32twzr3fi0y-50 storage/pbxt/src/pthread_xt.cc pthread_xt.cc-20090326121724-x683v32twzr3fi0y-51 storage/pbxt/src/restart_xt.cc restart_xt.cc-20090326121724-x683v32twzr3fi0y-53 storage/pbxt/src/restart_xt.h restart_xt.h-20090326121724-x683v32twzr3fi0y-54 storage/pbxt/src/sortedlist_xt.cc sortedlist_xt.cc-20090326121724-x683v32twzr3fi0y-55 storage/pbxt/src/strutil_xt.cc strutil_xt.cc-20090326121724-x683v32twzr3fi0y-59 storage/pbxt/src/systab_xt.cc systab_xt.cc-20090326121724-x683v32twzr3fi0y-61 storage/pbxt/src/systab_xt.h systab_xt.h-20090326121724-x683v32twzr3fi0y-62 storage/pbxt/src/tabcache_xt.cc tabcache_xt.cc-20090326121724-x683v32twzr3fi0y-63 storage/pbxt/src/tabcache_xt.h tabcache_xt.h-20090326121724-x683v32twzr3fi0y-64 storage/pbxt/src/table_xt.cc table_xt.cc-20090326121724-x683v32twzr3fi0y-65 storage/pbxt/src/table_xt.h table_xt.h-20090326121724-x683v32twzr3fi0y-66 storage/pbxt/src/thread_xt.cc thread_xt.cc-20090326121724-x683v32twzr3fi0y-67 storage/pbxt/src/thread_xt.h thread_xt.h-20090326121724-x683v32twzr3fi0y-68 storage/pbxt/src/trace_xt.cc trace_xt.cc-20090326121724-x683v32twzr3fi0y-69 storage/pbxt/src/trace_xt.h trace_xt.h-20090326121724-x683v32twzr3fi0y-70 storage/pbxt/src/util_xt.cc util_xt.cc-20090326121724-x683v32twzr3fi0y-71 storage/pbxt/src/xaction_xt.cc xaction_xt.cc-20090326121724-x683v32twzr3fi0y-74 storage/pbxt/src/xaction_xt.h xaction_xt.h-20090326121724-x683v32twzr3fi0y-75 storage/pbxt/src/xactlog_xt.cc xactlog_xt.cc-20090326121724-x683v32twzr3fi0y-76 storage/pbxt/src/xactlog_xt.h xactlog_xt.h-20090326121724-x683v32twzr3fi0y-77 storage/pbxt/src/xt_config.h xt_config.h-20090326121724-x683v32twzr3fi0y-78 storage/pbxt/src/xt_defs.h xt_defs.h-20090326121724-x683v32twzr3fi0y-79 storage/xtradb/Makefile.am makefile.am-20081201061010-zymrrwrczns2vrex-4 storage/xtradb/btr/btr0cur.c btr0cur.c-20081201061010-zymrrwrczns2vrex-45 storage/xtradb/buf/buf0buddy.c buf0buddy.c-20081201061010-zymrrwrczns2vrex-48 storage/xtradb/buf/buf0flu.c buf0flu.c-20081201061010-zymrrwrczns2vrex-50 storage/xtradb/fil/fil0fil.c fil0fil.c-20081201061010-zymrrwrczns2vrex-63 storage/xtradb/handler/i_s.cc i_s.cc-20081201061010-zymrrwrczns2vrex-73 storage/xtradb/srv/srv0srv.c srv0srv.c-20081201061010-zymrrwrczns2vrex-347 strings/CMakeLists.txt sp1f-cmakelists.txt-20060831175237-zpsnblto443qz4bb5okg23rsacehr4d2 unittest/mysys/waiting_threads-t.c waiting_threadst.c-20080623170213-r8baqa2porlpxzq1-5 ------------------------------------------------------------ revno: 2719.2.29 revision-id: psergey(a)askmonty.org-20090826210140-y2pba3ucdwnvr956 parent: psergey(a)askmonty.org-20090825150255-46i5gvl2ctd0jcm2 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11-gcov timestamp: Thu 2009-08-27 01:01:40 +0400 message: MWL#17: Table elimination: - Fix a trivial problem when OR-merging two multi-equalities - Amend testsuite to provide full gcov coverage modified: mysql-test/r/table_elim.result table_elim.result-20090603125022-nge13y0ohk1g2tt2-1 mysql-test/t/table_elim.test table_elim.test-20090603125018-ka3vcfrm07bsldz8-1 sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.28 revision-id: psergey(a)askmonty.org-20090825150255-46i5gvl2ctd0jcm2 parent: psergey(a)askmonty.org-20090825145915-rxwmfczgqe69r5ju committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11-gcov timestamp: Tue 2009-08-25 18:02:55 +0300 message: Gcov support improvements - add process-gcov-annotations.pl script that processes .gcov files according to purecov annotations in source code - add --gcov-src-dir=subdir option to mysql-test-run so that one can do gcov processing only for source files in a certain directory. added: mysql-test/lib/process-purecov-annotations.pl processpurecovannota-20090825145959-3tgucpo20ms8b5yb-1 modified: mysql-test/lib/mtr_gcov.pl sp1f-mtr_gcov.pl-20041230152646-g7ukx2aojk6vlfbrurijutdkqshwljue mysql-test/mysql-test-run.pl sp1f-mysqltestrun.pl-20041230152716-xjnn5ndv4rr4by6ijmj5a4ysubxc7qh3 ------------------------------------------------------------ revno: 2719.2.27 revision-id: psergey(a)askmonty.org-20090825145915-rxwmfczgqe69r5ju parent: psergey(a)askmonty.org-20090825103822-ldmjinfdh7ymc35r committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11-gcov timestamp: Tue 2009-08-25 17:59:15 +0300 message: MWL#17: Table elimination - Mark gcov deadcode modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.26 revision-id: psergey(a)askmonty.org-20090825103822-ldmjinfdh7ymc35r parent: psergey(a)askmonty.org-20090825092750-mnmatimn7cxm4rl8 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11-gcov timestamp: Tue 2009-08-25 13:38:22 +0300 message: MWL#17: Table elimination - More test coverage - Remove unused code modified: mysql-test/r/table_elim.result table_elim.result-20090603125022-nge13y0ohk1g2tt2-1 mysql-test/t/table_elim.test table_elim.test-20090603125018-ka3vcfrm07bsldz8-1 sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.25 revision-id: psergey(a)askmonty.org-20090825092750-mnmatimn7cxm4rl8 parent: psergey(a)askmonty.org-20090824191048-xev0gm1miw6ezz2r committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11-gcov timestamp: Tue 2009-08-25 12:27:50 +0300 message: MWL#17: Table elimination - Add more testcases. - Fix trivial compile failure - Remove handling of "column IN (one_element)". This is converted to equality elsewhere modified: mysql-test/r/table_elim.result table_elim.result-20090603125022-nge13y0ohk1g2tt2-1 mysql-test/t/table_elim.test table_elim.test-20090603125018-ka3vcfrm07bsldz8-1 sql/mysqld.cc sp1f-mysqld.cc-19700101030959-zpswdvekpvixxzxf7gdtofzel7nywtfj sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.24 revision-id: psergey(a)askmonty.org-20090824191048-xev0gm1miw6ezz2r parent: psergey(a)askmonty.org-20090824081242-32o90vv8awk27sut committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Mon 2009-08-24 22:10:48 +0300 message: MWL#17: Table elimination: last fixes - Add an @@optimizer_switch flag for table_elimination for debug build - Better comments modified: mysql-test/t/index_merge_myisam.test sp1f-index_merge_myisam.t-20060816114353-3p3p5fylcfj754j3d3td6zs6shnc5nu3 sql/mysql_priv.h sp1f-mysql_priv.h-19700101030959-4fl65tqpop5zfgxaxkqotu2fa2ree5ci sql/mysqld.cc sp1f-mysqld.cc-19700101030959-zpswdvekpvixxzxf7gdtofzel7nywtfj sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.23 revision-id: psergey(a)askmonty.org-20090824081242-32o90vv8awk27sut parent: psergey(a)askmonty.org-20090821133606-2t7hib7wuctqller committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Mon 2009-08-24 10:12:42 +0200 message: MWL#17: Table elimination - Correctly handle the case where we have multi-table DELETE and a table that we're deleting from looks like it could be eliminated. modified: mysql-test/r/table_elim.result table_elim.result-20090603125022-nge13y0ohk1g2tt2-1 mysql-test/t/table_elim.test table_elim.test-20090603125018-ka3vcfrm07bsldz8-1 sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.22 revision-id: psergey(a)askmonty.org-20090821133606-2t7hib7wuctqller parent: psergey(a)askmonty.org-20090821074822-6x2gv01r9ltt2bhc committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Fri 2009-08-21 15:36:06 +0200 message: MWL#17: Table elimination - Remove a piece of code that's not needed anymore. modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.21 revision-id: psergey(a)askmonty.org-20090821074822-6x2gv01r9ltt2bhc parent: psergey(a)askmonty.org-20090820155102-5zvm1m6idmie9mmv committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Fri 2009-08-21 09:48:22 +0200 message: MWL#17: Table elimination - More testcases - Set correct dependencies for non-bound multi-equalities. modified: mysql-test/r/table_elim.result table_elim.result-20090603125022-nge13y0ohk1g2tt2-1 mysql-test/t/table_elim.test table_elim.test-20090603125018-ka3vcfrm07bsldz8-1 sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.20 revision-id: psergey(a)askmonty.org-20090820155102-5zvm1m6idmie9mmv parent: psergey(a)askmonty.org-20090819120659-0ndd6mqxeetgee2n committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Thu 2009-08-20 17:51:02 +0200 message: MWL#17: Table elimination - Multiple-equality handling modified: sql/item_cmpfunc.h sp1f-item_cmpfunc.h-19700101030959-pcvbjplo4e4ng7ibynfhcd6pjyem57gr sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.19 revision-id: psergey(a)askmonty.org-20090819120659-0ndd6mqxeetgee2n parent: psergey(a)askmonty.org-20090819101838-55ys0h923olqz4q9 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Wed 2009-08-19 15:06:59 +0300 message: Variable/function renames modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.18 revision-id: psergey(a)askmonty.org-20090819101838-55ys0h923olqz4q9 parent: psergey(a)askmonty.org-20090818221948-f2mg0szfqrve06f9 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Wed 2009-08-19 13:18:38 +0300 message: MWL#17: Table elimination - Use Table_elimination only for functional dependency checking for individual objects and rename it to Func_dep_analyzer modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.17 revision-id: psergey(a)askmonty.org-20090818221948-f2mg0szfqrve06f9 parent: psergey(a)askmonty.org-20090818211810-48v6pb8dt0sqkysi committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Wed 2009-08-19 01:19:48 +0300 message: More code cleanups modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.16 revision-id: psergey(a)askmonty.org-20090818211810-48v6pb8dt0sqkysi parent: psergey(a)askmonty.org-20090818150151-uef38y50m8m1mgnu committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Wed 2009-08-19 00:18:10 +0300 message: MWL#17: Table elimination - Better comments - Switch from "type" enum and switch to virtual functions for member funcs. modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.15 revision-id: psergey(a)askmonty.org-20090818150151-uef38y50m8m1mgnu parent: psergey(a)askmonty.org-20090818130358-akd84j4m2i91lw5a committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Tue 2009-08-18 18:01:51 +0300 message: MWL#17: Table elimination - Code cleanup modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.14 revision-id: psergey(a)askmonty.org-20090818130358-akd84j4m2i91lw5a parent: psergey(a)askmonty.org-20090817160724-fmmrmwp8zorzn82q committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Tue 2009-08-18 16:03:58 +0300 message: MWL#17: Table elimination - Switch from trying to eliminate all tables at once (which didn't work) to the original approach of bottom-up elimination. modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.13 revision-id: psergey(a)askmonty.org-20090817160724-fmmrmwp8zorzn82q parent: psergey(a)askmonty.org-20090817150229-jy461nqbmk8nzhha committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Mon 2009-08-17 19:07:24 +0300 message: MWL#17: Table elimination - More testcases modified: mysql-test/r/table_elim.result table_elim.result-20090603125022-nge13y0ohk1g2tt2-1 mysql-test/t/table_elim.test table_elim.test-20090603125018-ka3vcfrm07bsldz8-1 ------------------------------------------------------------ revno: 2719.2.12 revision-id: psergey(a)askmonty.org-20090817150229-jy461nqbmk8nzhha parent: psergey(a)askmonty.org-20090816180159-z3lfkjpjfsm7zbp0 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Mon 2009-08-17 18:02:29 +0300 message: MWL#17: Table elimination - More dbug printouts - More testcases modified: mysql-test/r/table_elim.result table_elim.result-20090603125022-nge13y0ohk1g2tt2-1 mysql-test/t/table_elim.test table_elim.test-20090603125018-ka3vcfrm07bsldz8-1 sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.11 revision-id: psergey(a)askmonty.org-20090816180159-z3lfkjpjfsm7zbp0 parent: psergey(a)askmonty.org-20090816143547-16hyle50tbt31xen committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Sun 2009-08-16 21:01:59 +0300 message: MWL#17: Table elimination - More comments modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.10 revision-id: psergey(a)askmonty.org-20090816143547-16hyle50tbt31xen parent: psergey(a)askmonty.org-20090816124331-gd53m2alc0jb3ws4 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Sun 2009-08-16 17:35:47 +0300 message: MWL#17: Table elimination - Better comments - More OOM checks modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb ------------------------------------------------------------ revno: 2719.2.9 revision-id: psergey(a)askmonty.org-20090816124331-gd53m2alc0jb3ws4 parent: psergey(a)askmonty.org-20090816121708-v42h3mehvoy4c7yu committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Sun 2009-08-16 15:43:31 +0300 message: - Better comments - Add OOM error checking modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.8 revision-id: psergey(a)askmonty.org-20090816121708-v42h3mehvoy4c7yu parent: psergey(a)askmonty.org-20090816091549-da84w3nlmx8prmvm committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Sun 2009-08-16 15:17:08 +0300 message: MWL#17: Table elimination - Address review feedback: change expression analyzer used to be a copy-paste of ref analyzer. ref analyzer, besides doing ref analysis, also collected info about keys which had sargable predicates. We didn't need that part here. modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.7 revision-id: psergey(a)askmonty.org-20090816091549-da84w3nlmx8prmvm parent: psergey(a)askmonty.org-20090816072524-w9fu2hy23pjwlr8z committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Sun 2009-08-16 12:15:49 +0300 message: MWL#17: Table elimination - code cleanup modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.6 revision-id: psergey(a)askmonty.org-20090816072524-w9fu2hy23pjwlr8z parent: psergey(a)askmonty.org-20090815153912-q47vfp1j22ilmup2 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10-vg timestamp: Sun 2009-08-16 10:25:24 +0300 message: MWL#17: Table elimination - Fix trivial valgrind failures that shown up after review modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.5 revision-id: psergey(a)askmonty.org-20090815153912-q47vfp1j22ilmup2 parent: psergey(a)askmonty.org-20090815121442-706m9ujn8km4u4y1 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Sat 2009-08-15 18:39:12 +0300 message: MWL#17: Table elimination - Review feedback, more variable renames modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.4 revision-id: psergey(a)askmonty.org-20090815121442-706m9ujn8km4u4y1 parent: psergey(a)askmonty.org-20090815102953-7s0jb470ibwq58qz committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Sat 2009-08-15 16:14:42 +0400 message: Fix trivial typo modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.3 revision-id: psergey(a)askmonty.org-20090815102953-7s0jb470ibwq58qz parent: psergey(a)askmonty.org-20090815060803-0yvp5mmgo87emykp committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Sat 2009-08-15 14:29:53 +0400 message: MWL#17: Table elimination Continue with addressing review feedback part two: - rename enum members - add checking for out of memory errors on allocation modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.2 revision-id: psergey(a)askmonty.org-20090815060803-0yvp5mmgo87emykp parent: psergey(a)askmonty.org-20090813211212-jghejwxsl6adtopl committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Sat 2009-08-15 10:08:03 +0400 message: MWL#17: Address 2nd post-review feedback - Switch from uniform graph to bipartite graph with two kinds of nodes: "values" (tables and fields) and "modules" (t.col=func(...) equalities, multi-equalities, unique keys, inner sides of outer joins). - Rename functions, classes, etc. modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2719.2.1 revision-id: psergey(a)askmonty.org-20090813211212-jghejwxsl6adtopl parent: knielsen(a)knielsen-hq.org-20090805072137-wg97dcem1cxnzt3p parent: psergey(a)askmonty.org-20090813204452-o8whzlbio19cgkyv committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Fri 2009-08-14 01:12:12 +0400 message: Merge maria-5.1 -> maria-5.1-table-elimination added: mysql-test/r/table_elim.result table_elim.result-20090603125022-nge13y0ohk1g2tt2-1 mysql-test/t/table_elim.test table_elim.test-20090603125018-ka3vcfrm07bsldz8-1 sql-bench/test-table-elimination.sh testtableelimination-20090616194329-gai92muve732qknl-1 sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 modified: .bzrignore sp1f-ignore-20001018235455-q4gxfbritt5f42nwix354ufpsvrf5ebj libmysqld/Makefile.am sp1f-makefile.am-20010411110351-26htpk3ynkyh7pkfvnshztqrxx3few4g mysql-test/r/mysql-bug41486.result mysqlbug41486.result-20090323135900-fobg67a3yzg0b7e8-1 mysql-test/r/ps_11bugs.result sp1f-ps_11bugs.result-20041012140047-4pktjlfeq27q6bxqfdsbcszr5nybv6zz mysql-test/r/select.result sp1f-select.result-20010103001548-znkoalxem6wchsbxizfosjhpfmhfyxuk mysql-test/r/subselect.result sp1f-subselect.result-20020512204640-zgegcsgavnfd7t7eyrf7ibuqomsw7uzo mysql-test/r/union.result sp1f-unions_one.result-20010725122836-ofxtwraxeohz7whhrmfdz57sl4a5prmp mysql-test/t/mysql-bug41486.test mysqlbug41486.test-20090323135900-fobg67a3yzg0b7e8-2 mysql-test/valgrind.supp sp1f-valgrind.supp-20050406142216-yg7xhezklqhgqlc3inx36vbghodhbovy sql/CMakeLists.txt sp1f-cmakelists.txt-20060831175237-esoeu5kpdtwjvehkghwy6fzbleniq2wy sql/Makefile.am sp1f-makefile.am-19700101030959-xsjdiakci3nqcdd4xl4yomwdl5eo2f3q sql/item.cc sp1f-item.cc-19700101030959-u7hxqopwpfly4kf5ctlyk2dvrq4l3dhn sql/item.h sp1f-item.h-19700101030959-rrkb43htudd62batmoteashkebcwykpa sql/item_subselect.cc sp1f-item_subselect.cc-20020512204640-qep43aqhsfrwkqmrobni6czc3fqj36oo sql/item_subselect.h sp1f-item_subselect.h-20020512204640-qdg77wil56cxyhtc2bjjdrppxq3wqgh3 sql/item_sum.cc sp1f-item_sum.cc-19700101030959-4woo23bi3am2t2zvsddqbpxk7xbttdkm sql/item_sum.h sp1f-item_sum.h-19700101030959-ecgohlekwm355wxl5fv4zzq3alalbwyl sql/sql_bitmap.h sp1f-sql_bitmap.h-20031024204444-g4eiad7vopzqxe2trxmt3fn3xsvnomvj sql/sql_lex.cc sp1f-sql_lex.cc-19700101030959-4pizwlu5rqkti27gcwsvxkawq6bc2kph sql/sql_lex.h sp1f-sql_lex.h-19700101030959-sgldb2sooc7twtw5q7pgjx7qzqiaa3sn sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb sql/sql_select.h sp1f-sql_select.h-19700101030959-oqegfxr76xlgmrzd6qlevonoibfnwzoz sql/table.h sp1f-table.h-19700101030959-dv72bajftxj5fbdjuajquappanuv2ija ------------------------------------------------------------ revno: 2707.1.27 revision-id: psergey(a)askmonty.org-20090813204452-o8whzlbio19cgkyv parent: psergey(a)askmonty.org-20090813191053-g1xfeieoti4bqgbc committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r5 timestamp: Fri 2009-08-14 00:44:52 +0400 message: MWL#17: Table elimination - More function renames, added comments modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2707.1.26 revision-id: psergey(a)askmonty.org-20090813191053-g1xfeieoti4bqgbc parent: psergey(a)askmonty.org-20090813093613-hy7tdlsgdy83xszq committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r5 timestamp: Thu 2009-08-13 23:10:53 +0400 message: MWL#17: Table elimination - Better comments modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb ------------------------------------------------------------ revno: 2707.1.25 revision-id: psergey(a)askmonty.org-20090813093613-hy7tdlsgdy83xszq parent: psergey(a)askmonty.org-20090813092402-jlqucf6nultxlv4b committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r5 timestamp: Thu 2009-08-13 13:36:13 +0400 message: MWL#17: Table elimination Fixes after post-review fixes: - Don't search for tables in JOIN_TAB array. it's not initialized yet. use select_lex->leaf_tables instead. modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2707.1.24 revision-id: psergey(a)askmonty.org-20090813092402-jlqucf6nultxlv4b parent: psergey(a)askmonty.org-20090813000143-dukzk352hjywidk7 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r5 timestamp: Thu 2009-08-13 13:24:02 +0400 message: MWL#17: Table elimination - Post-postreview changes fix: Do set NESTED_JOIN::n_tables to number of tables left after elimination. modified: sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb ------------------------------------------------------------ revno: 2707.1.23 revision-id: psergey(a)askmonty.org-20090813000143-dukzk352hjywidk7 parent: psergey(a)askmonty.org-20090812234302-10es7qmf0m09ahbq committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r5 timestamp: Thu 2009-08-13 04:01:43 +0400 message: MWL#17: Table elimination - When making inferences "field is bound" -> "key is bound", do check that the field is part of the key modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2707.1.22 revision-id: psergey(a)askmonty.org-20090812234302-10es7qmf0m09ahbq parent: psergey(a)askmonty.org-20090812223421-w4xyzj7azqgo83ps committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r5 timestamp: Thu 2009-08-13 03:43:02 +0400 message: MWL#17: Table elimination - Continue addressing review feedback: remove "unusable KEYUSEs" extension as it is no longer needed. modified: sql/item.h sp1f-item.h-19700101030959-rrkb43htudd62batmoteashkebcwykpa sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb sql/sql_select.h sp1f-sql_select.h-19700101030959-oqegfxr76xlgmrzd6qlevonoibfnwzoz ------------------------------------------------------------ revno: 2707.1.21 revision-id: psergey(a)askmonty.org-20090812223421-w4xyzj7azqgo83ps parent: psergey(a)askmonty.org-20090708171038-9nyc3hcg1o7h8635 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r5 timestamp: Thu 2009-08-13 02:34:21 +0400 message: MWL#17: Table elimination Address review feedback: - Change from Wave-based approach (a-la const table detection) to building and walking functional dependency graph. - Change from piggy-backing on ref-access code and KEYUSE structures to using our own expression analyzer. modified: sql/item.cc sp1f-item.cc-19700101030959-u7hxqopwpfly4kf5ctlyk2dvrq4l3dhn sql/item.h sp1f-item.h-19700101030959-rrkb43htudd62batmoteashkebcwykpa sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 sql/sql_bitmap.h sp1f-sql_bitmap.h-20031024204444-g4eiad7vopzqxe2trxmt3fn3xsvnomvj ------------------------------------------------------------ revno: 2707.1.20 revision-id: psergey(a)askmonty.org-20090708171038-9nyc3hcg1o7h8635 parent: psergey(a)askmonty.org-20090630132018-8qwou8bqiq5z1qjg committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim timestamp: Wed 2009-07-08 21:10:38 +0400 message: MWL#17: Table elimination - When collecting Item_subselect::refers_to, put references to the correct subselect entry. modified: sql/sql_lex.cc sp1f-sql_lex.cc-19700101030959-4pizwlu5rqkti27gcwsvxkawq6bc2kph ------------------------------------------------------------ revno: 2707.1.19 revision-id: psergey(a)askmonty.org-20090630132018-8qwou8bqiq5z1qjg parent: psergey(a)askmonty.org-20090630131100-r6o8yqzse4yvny9l committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim timestamp: Tue 2009-06-30 17:20:18 +0400 message: MWL#17: Table elimination - More comments - Renove old code modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2707.1.18 revision-id: psergey(a)askmonty.org-20090630131100-r6o8yqzse4yvny9l parent: psergey(a)askmonty.org-20090629135115-472up9wsj0dq843i committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim timestamp: Tue 2009-06-30 17:11:00 +0400 message: MWL#17: Table elimination - Last fixes modified: sql/item.cc sp1f-item.cc-19700101030959-u7hxqopwpfly4kf5ctlyk2dvrq4l3dhn sql/item.h sp1f-item.h-19700101030959-rrkb43htudd62batmoteashkebcwykpa sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb sql/sql_select.h sp1f-sql_select.h-19700101030959-oqegfxr76xlgmrzd6qlevonoibfnwzoz sql/table.h sp1f-table.h-19700101030959-dv72bajftxj5fbdjuajquappanuv2ija ------------------------------------------------------------ revno: 2707.1.17 revision-id: psergey(a)askmonty.org-20090629135115-472up9wsj0dq843i parent: psergey(a)askmonty.org-20090625200729-u11xpwwn5ebddx09 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim timestamp: Mon 2009-06-29 17:51:15 +0400 message: MWL#17: Table elimination modified: mysql-test/r/table_elim.result table_elim.result-20090603125022-nge13y0ohk1g2tt2-1 mysql-test/t/table_elim.test table_elim.test-20090603125018-ka3vcfrm07bsldz8-1 sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb sql/sql_select.h sp1f-sql_select.h-19700101030959-oqegfxr76xlgmrzd6qlevonoibfnwzoz sql/table.h sp1f-table.h-19700101030959-dv72bajftxj5fbdjuajquappanuv2ija ------------------------------------------------------------ revno: 2707.1.16 revision-id: psergey(a)askmonty.org-20090625200729-u11xpwwn5ebddx09 parent: psergey(a)askmonty.org-20090625100947-mg9xwnbeyyjgzl3w committer: Sergey Petrunia <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-movearound timestamp: Fri 2009-06-26 00:07:29 +0400 message: MWL#17: Table elimination - Better comments, variable/function renames modified: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb sql/sql_select.h sp1f-sql_select.h-19700101030959-oqegfxr76xlgmrzd6qlevonoibfnwzoz ------------------------------------------------------------ revno: 2707.1.15 revision-id: psergey(a)askmonty.org-20090625100947-mg9xwnbeyyjgzl3w parent: psergey(a)askmonty.org-20090624224414-71xqbljy8jf4z1qs parent: psergey(a)askmonty.org-20090625100553-j1xenbz3o5nekiu2 committer: Sergey Petrunia <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim timestamp: Thu 2009-06-25 14:09:47 +0400 message: Automerge added: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 modified: .bzrignore sp1f-ignore-20001018235455-q4gxfbritt5f42nwix354ufpsvrf5ebj libmysqld/Makefile.am sp1f-makefile.am-20010411110351-26htpk3ynkyh7pkfvnshztqrxx3few4g sql/CMakeLists.txt sp1f-cmakelists.txt-20060831175237-esoeu5kpdtwjvehkghwy6fzbleniq2wy sql/Makefile.am sp1f-makefile.am-19700101030959-xsjdiakci3nqcdd4xl4yomwdl5eo2f3q sql/item.cc sp1f-item.cc-19700101030959-u7hxqopwpfly4kf5ctlyk2dvrq4l3dhn sql/item.h sp1f-item.h-19700101030959-rrkb43htudd62batmoteashkebcwykpa sql/item_subselect.cc sp1f-item_subselect.cc-20020512204640-qep43aqhsfrwkqmrobni6czc3fqj36oo sql/item_sum.h sp1f-item_sum.h-19700101030959-ecgohlekwm355wxl5fv4zzq3alalbwyl sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb sql/sql_select.h sp1f-sql_select.h-19700101030959-oqegfxr76xlgmrzd6qlevonoibfnwzoz ------------------------------------------------------------ revno: 2707.3.1 revision-id: psergey(a)askmonty.org-20090625100553-j1xenbz3o5nekiu2 parent: psergey(a)askmonty.org-20090624090104-c63mp3sfxcxytk0d committer: Sergey Petrunia <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-movearound timestamp: Thu 2009-06-25 14:05:53 +0400 message: MWL#17: Table elimination - Moved table elimination code to sql/opt_table_elimination.cc - Added comments added: sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 modified: .bzrignore sp1f-ignore-20001018235455-q4gxfbritt5f42nwix354ufpsvrf5ebj libmysqld/Makefile.am sp1f-makefile.am-20010411110351-26htpk3ynkyh7pkfvnshztqrxx3few4g sql/CMakeLists.txt sp1f-cmakelists.txt-20060831175237-esoeu5kpdtwjvehkghwy6fzbleniq2wy sql/Makefile.am sp1f-makefile.am-19700101030959-xsjdiakci3nqcdd4xl4yomwdl5eo2f3q sql/item.cc sp1f-item.cc-19700101030959-u7hxqopwpfly4kf5ctlyk2dvrq4l3dhn sql/item.h sp1f-item.h-19700101030959-rrkb43htudd62batmoteashkebcwykpa sql/item_subselect.cc sp1f-item_subselect.cc-20020512204640-qep43aqhsfrwkqmrobni6czc3fqj36oo sql/item_sum.h sp1f-item_sum.h-19700101030959-ecgohlekwm355wxl5fv4zzq3alalbwyl sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb sql/sql_select.h sp1f-sql_select.h-19700101030959-oqegfxr76xlgmrzd6qlevonoibfnwzoz ------------------------------------------------------------ revno: 2707.1.14 revision-id: psergey(a)askmonty.org-20090624224414-71xqbljy8jf4z1qs parent: psergey(a)askmonty.org-20090624090104-c63mp3sfxcxytk0d committer: Sergey Petrunia <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim timestamp: Thu 2009-06-25 02:44:14 +0400 message: MWL#17: Table elimination - fix a typo bug in has_eqref_access_candidate() - Adjust test to remove race condition modified: mysql-test/r/mysql-bug41486.result mysqlbug41486.result-20090323135900-fobg67a3yzg0b7e8-1 mysql-test/t/mysql-bug41486.test mysqlbug41486.test-20090323135900-fobg67a3yzg0b7e8-2 sql/item.cc sp1f-item.cc-19700101030959-u7hxqopwpfly4kf5ctlyk2dvrq4l3dhn ------------------------------------------------------------ revno: 2707.1.13 revision-id: psergey(a)askmonty.org-20090624090104-c63mp3sfxcxytk0d parent: psergey(a)askmonty.org-20090623200613-w9dl8g41ysf51r80 committer: Sergey Petrunia <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim timestamp: Wed 2009-06-24 13:01:04 +0400 message: More comments modified: sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb ------------------------------------------------------------ revno: 2707.1.12 revision-id: psergey(a)askmonty.org-20090623200613-w9dl8g41ysf51r80 parent: psergey(a)askmonty.org-20090622114631-yop0q2p8ktmfnctm committer: Sergey Petrunia <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim timestamp: Wed 2009-06-24 00:06:13 +0400 message: MWL#17: Table elimination - More testcases - Let add_ft_key() set keyuse->usable modified: mysql-test/r/table_elim.result table_elim.result-20090603125022-nge13y0ohk1g2tt2-1 mysql-test/t/table_elim.test table_elim.test-20090603125018-ka3vcfrm07bsldz8-1 sql-bench/test-table-elimination.sh testtableelimination-20090616194329-gai92muve732qknl-1 sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb ------------------------------------------------------------ revno: 2707.1.11 revision-id: psergey(a)askmonty.org-20090622114631-yop0q2p8ktmfnctm parent: psergey(a)askmonty.org-20090617052739-37i1r8lip0m4ft9r committer: Sergey Petrunia <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim timestamp: Mon 2009-06-22 15:46:31 +0400 message: MWL#17: Table elimination - Make elimination check to be able detect cases like t.primary_key_col1=othertbl.col AND t.primary_key_col2=func(t.primary_key_col1). These are needed to handle e.g. the case of func() being a correlated subquery that selects the latest value. - If we've removed a condition with subquery predicate, EXPLAIN [EXTENDED] won't show the subquery anymore modified: sql/item.cc sp1f-item.cc-19700101030959-u7hxqopwpfly4kf5ctlyk2dvrq4l3dhn sql/item.h sp1f-item.h-19700101030959-rrkb43htudd62batmoteashkebcwykpa sql/item_subselect.cc sp1f-item_subselect.cc-20020512204640-qep43aqhsfrwkqmrobni6czc3fqj36oo sql/item_subselect.h sp1f-item_subselect.h-20020512204640-qdg77wil56cxyhtc2bjjdrppxq3wqgh3 sql/item_sum.cc sp1f-item_sum.cc-19700101030959-4woo23bi3am2t2zvsddqbpxk7xbttdkm sql/sql_lex.cc sp1f-sql_lex.cc-19700101030959-4pizwlu5rqkti27gcwsvxkawq6bc2kph sql/sql_lex.h sp1f-sql_lex.h-19700101030959-sgldb2sooc7twtw5q7pgjx7qzqiaa3sn sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb sql/sql_select.h sp1f-sql_select.h-19700101030959-oqegfxr76xlgmrzd6qlevonoibfnwzoz ------------------------------------------------------------ revno: 2707.1.10 revision-id: psergey(a)askmonty.org-20090617052739-37i1r8lip0m4ft9r parent: psergey(a)askmonty.org-20090616204358-yjkyfxczsomrn9yn committer: Sergey Petrunia <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim timestamp: Wed 2009-06-17 09:27:39 +0400 message: * Use excessive parentheses to stop compiler warning * Fix test results to account for changes in previous cset modified: mysql-test/r/select.result sp1f-select.result-20010103001548-znkoalxem6wchsbxizfosjhpfmhfyxuk sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb ------------------------------------------------------------ revno: 2707.1.9 revision-id: psergey(a)askmonty.org-20090616204358-yjkyfxczsomrn9yn parent: psergey(a)askmonty.org-20090616195413-rfmi9un20za8gn8g parent: psergey(a)askmonty.org-20090615162208-p4w8s8jo06bdz1vj committer: Sergey Petrunia <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim timestamp: Wed 2009-06-17 00:43:58 +0400 message: * Merge * Change valgrind suppression to work on valgrind 3.3.0 modified: mysql-test/valgrind.supp sp1f-valgrind.supp-20050406142216-yg7xhezklqhgqlc3inx36vbghodhbovy ------------------------------------------------------------ revno: 2707.2.1 revision-id: psergey(a)askmonty.org-20090615162208-p4w8s8jo06bdz1vj parent: psergey(a)askmonty.org-20090614205924-1vnfwbuo4brzyfhp committer: Sergey Petrunia <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-movearound timestamp: Mon 2009-06-15 20:22:08 +0400 message: Fix spurious valgrind warnings in rpl_trigger.test modified: mysql-test/valgrind.supp sp1f-valgrind.supp-20050406142216-yg7xhezklqhgqlc3inx36vbghodhbovy ------------------------------------------------------------ revno: 2707.1.8 revision-id: psergey(a)askmonty.org-20090616195413-rfmi9un20za8gn8g parent: psergey(a)askmonty.org-20090614205924-1vnfwbuo4brzyfhp committer: Sergey Petrunia <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim timestamp: Tue 2009-06-16 23:54:13 +0400 message: MWL#17: Table elimination - Move eliminate_tables() to before constant table detection. - First code for benchmark added: sql-bench/test-table-elimination.sh testtableelimination-20090616194329-gai92muve732qknl-1 modified: sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb ------------------------------------------------------------ revno: 2707.1.7 revision-id: psergey(a)askmonty.org-20090614205924-1vnfwbuo4brzyfhp parent: psergey(a)askmonty.org-20090614123504-jf4pcb333ojwaxfy committer: Sergey Petrunia <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim timestamp: Mon 2009-06-15 00:59:24 +0400 message: MWL#17: Table elimination - Fix print_join() to work both for EXPLAIN EXTENDED (after table elimination) and for CREATE VIEW (after join->prepare() but without any optimization). modified: mysql-test/r/union.result sp1f-unions_one.result-20010725122836-ofxtwraxeohz7whhrmfdz57sl4a5prmp sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb ------------------------------------------------------------ revno: 2707.1.6 revision-id: psergey(a)askmonty.org-20090614123504-jf4pcb333ojwaxfy parent: psergey(a)askmonty.org-20090614100110-u7l54gk0b6zbtj50 committer: Sergey Petrunia <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim timestamp: Sun 2009-06-14 16:35:04 +0400 message: MWL#17: Table elimination - Fix the previous cset: take into account that select_lex may be printed when 1. There is no select_lex->join at all (in that case, assume that no tables were eliminated) 2. select_lex->join exists but there was no JOIN::optimize() call yet. handle this by initializing join->eliminated really early. modified: sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb sql/sql_select.h sp1f-sql_select.h-19700101030959-oqegfxr76xlgmrzd6qlevonoibfnwzoz ------------------------------------------------------------ revno: 2707.1.5 revision-id: psergey(a)askmonty.org-20090614100110-u7l54gk0b6zbtj50 parent: psergey(a)askmonty.org-20090609211133-wfau2tgwo2vpgc5d committer: Sergey Petrunia <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim timestamp: Sun 2009-06-14 14:01:10 +0400 message: MWL#17: Table elimination - Do not show eliminated tables in the output of EXPLAIN EXTENDED modified: mysql-test/r/table_elim.result table_elim.result-20090603125022-nge13y0ohk1g2tt2-1 mysql-test/t/table_elim.test table_elim.test-20090603125018-ka3vcfrm07bsldz8-1 sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb sql/sql_select.h sp1f-sql_select.h-19700101030959-oqegfxr76xlgmrzd6qlevonoibfnwzoz sql/table.h sp1f-table.h-19700101030959-dv72bajftxj5fbdjuajquappanuv2ija ------------------------------------------------------------ revno: 2707.1.4 revision-id: psergey(a)askmonty.org-20090609211133-wfau2tgwo2vpgc5d parent: psergey(a)askmonty.org-20090608135546-ut1yrzbah4gdw6e6 committer: Sergey Petrunia <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim timestamp: Wed 2009-06-10 01:11:33 +0400 message: MWL#17: Table elimination - Make elimination work with aggregate functions. The problem was that aggregate functions reported all table bits in used_tables(), and that prevented table elimination. Fixed by making aggregate functions return more correct value from used_tables(). modified: mysql-test/r/ps_11bugs.result sp1f-ps_11bugs.result-20041012140047-4pktjlfeq27q6bxqfdsbcszr5nybv6zz mysql-test/r/subselect.result sp1f-subselect.result-20020512204640-zgegcsgavnfd7t7eyrf7ibuqomsw7uzo mysql-test/r/table_elim.result table_elim.result-20090603125022-nge13y0ohk1g2tt2-1 mysql-test/t/table_elim.test table_elim.test-20090603125018-ka3vcfrm07bsldz8-1 sql/item.h sp1f-item.h-19700101030959-rrkb43htudd62batmoteashkebcwykpa sql/item_sum.cc sp1f-item_sum.cc-19700101030959-4woo23bi3am2t2zvsddqbpxk7xbttdkm sql/item_sum.h sp1f-item_sum.h-19700101030959-ecgohlekwm355wxl5fv4zzq3alalbwyl ------------------------------------------------------------ revno: 2707.1.3 revision-id: psergey(a)askmonty.org-20090608135546-ut1yrzbah4gdw6e6 parent: psergey(a)askmonty.org-20090607182938-ycajee5ozg33b7c8 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-fix timestamp: Mon 2009-06-08 17:55:46 +0400 message: Fix valgrind failure: provide an implementation of strmov_overlapp() that really can handle overlapping. added: strings/strmov_overlapp.c strmov_overlapp.c-20090608135132-403c5p4dlnexqwxi-1 modified: include/m_string.h sp1f-m_string.h-19700101030959-rraattbvw5ffkokv4sixxf3s7brqqaga libmysql/Makefile.shared sp1f-makefile.shared-20000818182429-m3kdhxi23vorlqjct2y2hl3yw357jtxt strings/Makefile.am sp1f-makefile.am-19700101030959-jfitkanzc3r4h2otoyaaprgqn7muf4ux ------------------------------------------------------------ revno: 2707.1.2 revision-id: psergey(a)askmonty.org-20090607182938-ycajee5ozg33b7c8 parent: psergey(a)askmonty.org-20090603182330-ll3gc91iowhtgb23 parent: psergey(a)askmonty.org-20090607182403-6sfpvdr7nkkekcy9 committer: Sergey Petrunia <psergey(a)askmonty.org> branch nick: maria-5.1 timestamp: Sun 2009-06-07 22:29:38 +0400 message: Merge MWL#17: Table elimination modified: sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb ------------------------------------------------------------ revno: 2705.2.2 revision-id: psergey(a)askmonty.org-20090607182403-6sfpvdr7nkkekcy9 parent: psergey(a)askmonty.org-20090603131045-c8jqhwlanli7eimv committer: Sergey Petrunia <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim timestamp: Sun 2009-06-07 22:24:03 +0400 message: MWL#17: Table Elimination - Fix trivial valgrind warning modified: sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb ------------------------------------------------------------ revno: 2707.1.1 revision-id: psergey(a)askmonty.org-20090603182330-ll3gc91iowhtgb23 parent: knielsen(a)knielsen-hq.org-20090602110359-n4q9gof38buucrny parent: psergey(a)askmonty.org-20090603131045-c8jqhwlanli7eimv committer: Sergey Petrunia <psergey(a)askmonty.org> branch nick: maria-5.1 timestamp: Wed 2009-06-03 22:23:30 +0400 message: Merge MWL#17 with maria/5.1 added: mysql-test/r/table_elim.result table_elim.result-20090603125022-nge13y0ohk1g2tt2-1 mysql-test/t/table_elim.test table_elim.test-20090603125018-ka3vcfrm07bsldz8-1 modified: sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb sql/sql_select.h sp1f-sql_select.h-19700101030959-oqegfxr76xlgmrzd6qlevonoibfnwzoz sql/table.h sp1f-table.h-19700101030959-dv72bajftxj5fbdjuajquappanuv2ija ------------------------------------------------------------ revno: 2705.2.1 revision-id: psergey(a)askmonty.org-20090603131045-c8jqhwlanli7eimv parent: knielsen(a)knielsen-hq.org-20090522175325-xpwm83ilnhqoqjz0 committer: Sergey Petrunia <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim timestamp: Wed 2009-06-03 17:10:45 +0400 message: MWL#17: Table elimination - First code. Elimination works for simple cases, passes the testsuite. - Known issues: = No elimination is done for aggregate functions. = EXPLAIN EXTENDED shows eliminated tables (I think it better not) = No benchmark yet = The code needs some polishing. added: mysql-test/r/table_elim.result table_elim.result-20090603125022-nge13y0ohk1g2tt2-1 mysql-test/t/table_elim.test table_elim.test-20090603125018-ka3vcfrm07bsldz8-1 modified: sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb sql/sql_select.h sp1f-sql_select.h-19700101030959-oqegfxr76xlgmrzd6qlevonoibfnwzoz sql/table.h sp1f-table.h-19700101030959-dv72bajftxj5fbdjuajquappanuv2ija Diff too large for email (4431 lines, the limit is 1000).

1 0

[Maria-developers] Rev 2724: More comments in file:///home/psergey/dev/maria-5.1/
by Sergey Petrunya 02 Sep '09

02 Sep '09

At file:///home/psergey/dev/maria-5.1/ ------------------------------------------------------------ revno: 2724 revision-id: psergey(a)askmonty.org-20090902084018-lv3qet4nhooc4cnu parent: psergey(a)askmonty.org-20090901115954-nge2brzh3akkmtlz committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1 timestamp: Wed 2009-09-02 12:40:18 +0400 message: More comments === modified file 'sql/opt_range.h' --- a/sql/opt_range.h 2009-05-06 12:03:24 +0000 +++ b/sql/opt_range.h 2009-09-02 08:40:18 +0000 @@ -38,6 +38,12 @@ } KEY_PART; +/* + A "MIN_TUPLE < tbl.key_tuple < MAX_TUPLE" interval. + + One of endpoints may be absent. 'flags' member has flags which tell whether + the endpoints are '<' or '<='. +*/ class QUICK_RANGE :public Sql_alloc { public: uchar *min_key,*max_key;

1 0

[Maria-developers] Rev 2750: Merge in file:///home/psergey/dev/maria-5.1-table-elim-r12/
by Sergey Petrunya 02 Sep '09

02 Sep '09

At file:///home/psergey/dev/maria-5.1-table-elim-r12/ ------------------------------------------------------------ revno: 2750 revision-id: psergey(a)askmonty.org-20090901222415-c3142zs9nstn2af0 parent: psergey(a)askmonty.org-20090901222000-st4iicaxopeex2s1 parent: psergey(a)askmonty.org-20090901214116-n5bzyqlhgl0lamno committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r12 timestamp: Wed 2009-09-02 02:24:15 +0400 message: Merge modified: include/my_global.h sp1f-my_global.h-20010915021246-4vawdgfw4vg3tuxq6mejt7lrchcnceha sql-bench/test-table-elimination.sh testtableelimination-20090616194329-gai92muve732qknl-1 sql/item.cc sp1f-item.cc-19700101030959-u7hxqopwpfly4kf5ctlyk2dvrq4l3dhn sql/item.h sp1f-item.h-19700101030959-rrkb43htudd62batmoteashkebcwykpa sql/item_cmpfunc.cc sp1f-item_cmpfunc.cc-19700101030959-hrk7pi2n6qpwxauufnkizirsoucdcx2e sql/item_subselect.cc sp1f-item_subselect.cc-20020512204640-qep43aqhsfrwkqmrobni6czc3fqj36oo sql/item_subselect.h sp1f-item_subselect.h-20020512204640-qdg77wil56cxyhtc2bjjdrppxq3wqgh3 sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 sql/sql_list.h sp1f-sql_list.h-19700101030959-hyddr5nvvtrgbk7llgnah4i4bf6ugspn sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb ------------------------------------------------------------ revno: 2748.1.2 revision-id: psergey(a)askmonty.org-20090901214116-n5bzyqlhgl0lamno parent: psergey(a)askmonty.org-20090831200209-7gbnkr7omjbgld1x committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Wed 2009-09-02 01:41:16 +0400 message: MWL#17: Table elimination - Address review feedback R4: better comments, formatting modified: sql-bench/test-table-elimination.sh testtableelimination-20090616194329-gai92muve732qknl-1 sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 ------------------------------------------------------------ revno: 2748.1.1 revision-id: psergey(a)askmonty.org-20090831200209-7gbnkr7omjbgld1x parent: psergey(a)askmonty.org-20090826210140-y2pba3ucdwnvr956 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Tue 2009-09-01 00:02:09 +0400 message: MWL#17: Table-elimination - Addressing review feedback, generation 4. modified: include/my_global.h sp1f-my_global.h-20010915021246-4vawdgfw4vg3tuxq6mejt7lrchcnceha sql/item.cc sp1f-item.cc-19700101030959-u7hxqopwpfly4kf5ctlyk2dvrq4l3dhn sql/item.h sp1f-item.h-19700101030959-rrkb43htudd62batmoteashkebcwykpa sql/item_cmpfunc.cc sp1f-item_cmpfunc.cc-19700101030959-hrk7pi2n6qpwxauufnkizirsoucdcx2e sql/item_subselect.cc sp1f-item_subselect.cc-20020512204640-qep43aqhsfrwkqmrobni6czc3fqj36oo sql/item_subselect.h sp1f-item_subselect.h-20020512204640-qdg77wil56cxyhtc2bjjdrppxq3wqgh3 sql/opt_table_elimination.cc opt_table_eliminatio-20090625095316-7ka9w3zr7n5114iv-1 sql/sql_list.h sp1f-sql_list.h-19700101030959-hyddr5nvvtrgbk7llgnah4i4bf6ugspn sql/sql_select.cc sp1f-sql_select.cc-19700101030959-egb7whpkh76zzvikycs5nsnuviu4fdlb Diff too large for email (2729 lines, the limit is 1000).

1 0

[Maria-developers] Rev 2749: Merge in file:///home/psergey/dev/maria-5.1-table-elim-r12/
by Sergey Petrunya 02 Sep '09

02 Sep '09

At file:///home/psergey/dev/maria-5.1-table-elim-r12/ ------------------------------------------------------------ revno: 2749 revision-id: psergey(a)askmonty.org-20090901222000-st4iicaxopeex2s1 parent: psergey(a)askmonty.org-20090826210140-y2pba3ucdwnvr956 parent: psergey(a)askmonty.org-20090901115954-nge2brzh3akkmtlz committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r12 timestamp: Wed 2009-09-02 02:20:00 +0400 message: Merge removed: storage/pbxt/src/streaming_xt.cc streaming_xt.cc-20090326121724-x683v32twzr3fi0y-57 storage/pbxt/src/streaming_xt.h streaming_xt.h-20090326121724-x683v32twzr3fi0y-58 added: mysql-test/suite/pbxt/t/lowercase_table_grant-master.opt lowercase_table_gran-20090828130852-3duj73io0g345hxj-1 mysql-test/suite/pbxt/t/lowercase_table_qcache-master.opt lowercase_table_qcac-20090828130852-3duj73io0g345hxj-2 mysql-test/suite/pbxt/t/lowercase_view-master.opt lowercase_viewmaster-20090828130852-3duj73io0g345hxj-3 mysql-test/suite/pbxt/t/udf-master.opt udfmaster.opt-20090828130852-3duj73io0g345hxj-4 storage/pbxt/src/pbms_enabled.cc pbms_enabled.cc-20090818074502-tybcb62hp1kdrk3l-1 storage/pbxt/src/pbms_enabled.h pbms_enabled.h-20090818074502-tybcb62hp1kdrk3l-2 modified: BUILD/Makefile.am sp1f-makefile.am-20020102192940-dza66ux2yxyklupzjz4q3km3hvye5rnj libmysql/CMakeLists.txt sp1f-cmakelists.txt-20060831175237-3bbprkzqdl3gopcnlt3evmaikau67bcq mysql-test/suite/pbxt/r/alter_table.result alter_table.result-20090402100035-4ilk9i91sh65vjcb-5 mysql-test/suite/pbxt/r/analyze.result analyze.result-20090402100035-4ilk9i91sh65vjcb-7 mysql-test/suite/pbxt/r/auto_increment.result auto_increment.resul-20090402100035-4ilk9i91sh65vjcb-9 mysql-test/suite/pbxt/r/delete.result delete.result-20090402100035-4ilk9i91sh65vjcb-31 mysql-test/suite/pbxt/r/distinct.result distinct.result-20090402100035-4ilk9i91sh65vjcb-34 mysql-test/suite/pbxt/r/func_group.result func_group.result-20090402100035-4ilk9i91sh65vjcb-50 mysql-test/suite/pbxt/r/func_math.result func_math.result-20090402100035-4ilk9i91sh65vjcb-55 mysql-test/suite/pbxt/r/func_str.result func_str.result-20090402100035-4ilk9i91sh65vjcb-61 mysql-test/suite/pbxt/r/grant.result grant.result-20090402100035-4ilk9i91sh65vjcb-65 mysql-test/suite/pbxt/r/group_min_max.result group_min_max.result-20090402100035-4ilk9i91sh65vjcb-69 mysql-test/suite/pbxt/r/join.result join.result-20090402100035-4ilk9i91sh65vjcb-79 mysql-test/suite/pbxt/r/join_nested.result join_nested.result-20090402100035-4ilk9i91sh65vjcb-81 mysql-test/suite/pbxt/r/key.result key.result-20090402100035-4ilk9i91sh65vjcb-83 mysql-test/suite/pbxt/r/key_cache.result key_cache.result-20090402100035-4ilk9i91sh65vjcb-84 mysql-test/suite/pbxt/r/key_diff.result key_diff.result-20090402100035-4ilk9i91sh65vjcb-85 mysql-test/suite/pbxt/r/lowercase_view.result lowercase_view.resul-20090402100035-4ilk9i91sh65vjcb-96 mysql-test/suite/pbxt/r/mysqlshow.result mysqlshow.result-20090402100035-4ilk9i91sh65vjcb-101 mysql-test/suite/pbxt/r/null.result null.result-20090402100035-4ilk9i91sh65vjcb-104 mysql-test/suite/pbxt/r/null_key.result null_key.result-20090402100035-4ilk9i91sh65vjcb-105 mysql-test/suite/pbxt/r/partition_pruning.result partition_pruning.re-20090402100035-4ilk9i91sh65vjcb-117 mysql-test/suite/pbxt/r/pbxt_bugs.result pbxt_bugs.result-20090402100035-4ilk9i91sh65vjcb-120 mysql-test/suite/pbxt/r/pbxt_ref_int.result pbxt_ref_int.result-20090402100035-4ilk9i91sh65vjcb-123 mysql-test/suite/pbxt/r/preload.result preload.result-20090402100035-4ilk9i91sh65vjcb-127 mysql-test/suite/pbxt/r/ps_1general.result ps_1general.result-20090402100035-4ilk9i91sh65vjcb-130 mysql-test/suite/pbxt/r/range.result range.result-20090402100035-4ilk9i91sh65vjcb-132 mysql-test/suite/pbxt/r/schema.result schema.result-20090402100035-4ilk9i91sh65vjcb-138 mysql-test/suite/pbxt/r/select.result select.result-20090402100035-4ilk9i91sh65vjcb-139 mysql-test/suite/pbxt/r/select_safe.result select_safe.result-20090402100035-4ilk9i91sh65vjcb-141 mysql-test/suite/pbxt/r/subselect.result subselect.result-20090402100035-4ilk9i91sh65vjcb-146 mysql-test/suite/pbxt/r/type_enum.result type_enum.result-20090402100035-4ilk9i91sh65vjcb-155 mysql-test/suite/pbxt/r/type_ranges.result type_ranges.result-20090402100035-4ilk9i91sh65vjcb-159 mysql-test/suite/pbxt/r/type_timestamp.result type_timestamp.resul-20090402100035-4ilk9i91sh65vjcb-162 mysql-test/suite/pbxt/r/union.result union.result-20090402100035-4ilk9i91sh65vjcb-166 mysql-test/suite/pbxt/r/view_grant.result view_grant.result-20090402100035-4ilk9i91sh65vjcb-169 mysql-test/suite/pbxt/t/auto_increment.test auto_increment.test-20090402100035-4ilk9i91sh65vjcb-176 mysql-test/suite/pbxt/t/delete.test delete.test-20090402100035-4ilk9i91sh65vjcb-198 mysql-test/suite/pbxt/t/join_nested.test join_nested.test-20090402100035-4ilk9i91sh65vjcb-248 mysql-test/suite/pbxt/t/null.test null.test-20090402100035-4ilk9i91sh65vjcb-271 mysql-test/suite/pbxt/t/pbxt_bugs.test pbxt_bugs.test-20090402100035-4ilk9i91sh65vjcb-287 mysql-test/suite/pbxt/t/rename.test rename.test-20090402100035-4ilk9i91sh65vjcb-301 mysql-test/suite/pbxt/t/schema.test schema.test-20090402100035-4ilk9i91sh65vjcb-305 mysql-test/suite/pbxt/t/type_enum.test type_enum.test-20090402100035-4ilk9i91sh65vjcb-322 mysql-test/suite/pbxt/t/union.test union.test-20090402100035-4ilk9i91sh65vjcb-333 sql/CMakeLists.txt sp1f-cmakelists.txt-20060831175237-esoeu5kpdtwjvehkghwy6fzbleniq2wy storage/pbxt/ChangeLog changelog-20090326121724-x683v32twzr3fi0y-3 storage/pbxt/src/Makefile.am makefile.am-20090326121724-x683v32twzr3fi0y-13 storage/pbxt/src/cache_xt.cc cache_xt.cc-20090326121724-x683v32twzr3fi0y-16 storage/pbxt/src/cache_xt.h cache_xt.h-20090326121724-x683v32twzr3fi0y-17 storage/pbxt/src/ccutils_xt.cc ccutils_xt.cc-20090326121724-x683v32twzr3fi0y-18 storage/pbxt/src/database_xt.cc database_xt.cc-20090326121724-x683v32twzr3fi0y-20 storage/pbxt/src/datadic_xt.cc datadic_xt.cc-20090326121724-x683v32twzr3fi0y-22 storage/pbxt/src/datadic_xt.h datadic_xt.h-20090326121724-x683v32twzr3fi0y-23 storage/pbxt/src/datalog_xt.cc datalog_xt.cc-20090326121724-x683v32twzr3fi0y-24 storage/pbxt/src/datalog_xt.h datalog_xt.h-20090326121724-x683v32twzr3fi0y-25 storage/pbxt/src/discover_xt.cc discover_xt.cc-20090326121724-x683v32twzr3fi0y-26 storage/pbxt/src/filesys_xt.cc filesys_xt.cc-20090326121724-x683v32twzr3fi0y-28 storage/pbxt/src/filesys_xt.h filesys_xt.h-20090326121724-x683v32twzr3fi0y-29 storage/pbxt/src/ha_pbxt.cc ha_pbxt.cc-20090326121724-x683v32twzr3fi0y-30 storage/pbxt/src/ha_pbxt.h ha_pbxt.h-20090326121724-x683v32twzr3fi0y-31 storage/pbxt/src/ha_xtsys.cc ha_xtsys.cc-20090326121724-x683v32twzr3fi0y-32 storage/pbxt/src/ha_xtsys.h ha_xtsys.h-20090326121724-x683v32twzr3fi0y-33 storage/pbxt/src/hashtab_xt.cc hashtab_xt.cc-20090326121724-x683v32twzr3fi0y-34 storage/pbxt/src/heap_xt.cc heap_xt.cc-20090326121724-x683v32twzr3fi0y-36 storage/pbxt/src/heap_xt.h heap_xt.h-20090326121724-x683v32twzr3fi0y-37 storage/pbxt/src/index_xt.cc index_xt.cc-20090326121724-x683v32twzr3fi0y-38 storage/pbxt/src/index_xt.h index_xt.h-20090326121724-x683v32twzr3fi0y-39 storage/pbxt/src/lock_xt.cc lock_xt.cc-20090326121724-x683v32twzr3fi0y-42 storage/pbxt/src/lock_xt.h lock_xt.h-20090326121724-x683v32twzr3fi0y-43 storage/pbxt/src/locklist_xt.cc locklist_xt.cc-20090326121724-x683v32twzr3fi0y-44 storage/pbxt/src/locklist_xt.h locklist_xt.h-20090326121724-x683v32twzr3fi0y-45 storage/pbxt/src/memory_xt.cc memory_xt.cc-20090326121724-x683v32twzr3fi0y-46 storage/pbxt/src/memory_xt.h memory_xt.h-20090326121724-x683v32twzr3fi0y-47 storage/pbxt/src/myxt_xt.cc myxt_xt.cc-20090326121724-x683v32twzr3fi0y-48 storage/pbxt/src/myxt_xt.h myxt_xt.h-20090326121724-x683v32twzr3fi0y-49 storage/pbxt/src/pbms.h pbms.h-20090326121724-x683v32twzr3fi0y-50 storage/pbxt/src/pthread_xt.cc pthread_xt.cc-20090326121724-x683v32twzr3fi0y-51 storage/pbxt/src/restart_xt.cc restart_xt.cc-20090326121724-x683v32twzr3fi0y-53 storage/pbxt/src/restart_xt.h restart_xt.h-20090326121724-x683v32twzr3fi0y-54 storage/pbxt/src/sortedlist_xt.cc sortedlist_xt.cc-20090326121724-x683v32twzr3fi0y-55 storage/pbxt/src/strutil_xt.cc strutil_xt.cc-20090326121724-x683v32twzr3fi0y-59 storage/pbxt/src/systab_xt.cc systab_xt.cc-20090326121724-x683v32twzr3fi0y-61 storage/pbxt/src/systab_xt.h systab_xt.h-20090326121724-x683v32twzr3fi0y-62 storage/pbxt/src/tabcache_xt.cc tabcache_xt.cc-20090326121724-x683v32twzr3fi0y-63 storage/pbxt/src/tabcache_xt.h tabcache_xt.h-20090326121724-x683v32twzr3fi0y-64 storage/pbxt/src/table_xt.cc table_xt.cc-20090326121724-x683v32twzr3fi0y-65 storage/pbxt/src/table_xt.h table_xt.h-20090326121724-x683v32twzr3fi0y-66 storage/pbxt/src/thread_xt.cc thread_xt.cc-20090326121724-x683v32twzr3fi0y-67 storage/pbxt/src/thread_xt.h thread_xt.h-20090326121724-x683v32twzr3fi0y-68 storage/pbxt/src/trace_xt.cc trace_xt.cc-20090326121724-x683v32twzr3fi0y-69 storage/pbxt/src/trace_xt.h trace_xt.h-20090326121724-x683v32twzr3fi0y-70 storage/pbxt/src/util_xt.cc util_xt.cc-20090326121724-x683v32twzr3fi0y-71 storage/pbxt/src/xaction_xt.cc xaction_xt.cc-20090326121724-x683v32twzr3fi0y-74 storage/pbxt/src/xaction_xt.h xaction_xt.h-20090326121724-x683v32twzr3fi0y-75 storage/pbxt/src/xactlog_xt.cc xactlog_xt.cc-20090326121724-x683v32twzr3fi0y-76 storage/pbxt/src/xactlog_xt.h xactlog_xt.h-20090326121724-x683v32twzr3fi0y-77 storage/pbxt/src/xt_config.h xt_config.h-20090326121724-x683v32twzr3fi0y-78 storage/pbxt/src/xt_defs.h xt_defs.h-20090326121724-x683v32twzr3fi0y-79 storage/xtradb/Makefile.am makefile.am-20081201061010-zymrrwrczns2vrex-4 storage/xtradb/btr/btr0cur.c btr0cur.c-20081201061010-zymrrwrczns2vrex-45 storage/xtradb/buf/buf0buddy.c buf0buddy.c-20081201061010-zymrrwrczns2vrex-48 storage/xtradb/buf/buf0flu.c buf0flu.c-20081201061010-zymrrwrczns2vrex-50 storage/xtradb/fil/fil0fil.c fil0fil.c-20081201061010-zymrrwrczns2vrex-63 storage/xtradb/handler/i_s.cc i_s.cc-20081201061010-zymrrwrczns2vrex-73 storage/xtradb/srv/srv0srv.c srv0srv.c-20081201061010-zymrrwrczns2vrex-347 strings/CMakeLists.txt sp1f-cmakelists.txt-20060831175237-zpsnblto443qz4bb5okg23rsacehr4d2 unittest/mysys/waiting_threads-t.c waiting_threadst.c-20080623170213-r8baqa2porlpxzq1-5 ------------------------------------------------------------ revno: 2719.1.4 revision-id: psergey(a)askmonty.org-20090901115954-nge2brzh3akkmtlz parent: knielsen(a)knielsen-hq.org-20090831112339-td6t7ljy4w2oamvt committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-fixwindows timestamp: Tue 2009-09-01 15:59:54 +0400 message: Trivial fixes for windows compile failures - Add strmov_overlapp.c where it's needed - Add libevent where it is required - Fix typo in unittest/mysys/waiting_threads-t.c - fix xtradb C language source so that they have variable declarations before any operators, as it is (or was?) required in C. modified: libmysql/CMakeLists.txt sp1f-cmakelists.txt-20060831175237-3bbprkzqdl3gopcnlt3evmaikau67bcq sql/CMakeLists.txt sp1f-cmakelists.txt-20060831175237-esoeu5kpdtwjvehkghwy6fzbleniq2wy storage/xtradb/btr/btr0cur.c btr0cur.c-20081201061010-zymrrwrczns2vrex-45 storage/xtradb/buf/buf0buddy.c buf0buddy.c-20081201061010-zymrrwrczns2vrex-48 storage/xtradb/buf/buf0flu.c buf0flu.c-20081201061010-zymrrwrczns2vrex-50 storage/xtradb/fil/fil0fil.c fil0fil.c-20081201061010-zymrrwrczns2vrex-63 storage/xtradb/handler/i_s.cc i_s.cc-20081201061010-zymrrwrczns2vrex-73 storage/xtradb/srv/srv0srv.c srv0srv.c-20081201061010-zymrrwrczns2vrex-347 strings/CMakeLists.txt sp1f-cmakelists.txt-20060831175237-zpsnblto443qz4bb5okg23rsacehr4d2 unittest/mysys/waiting_threads-t.c waiting_threadst.c-20080623170213-r8baqa2porlpxzq1-5 ------------------------------------------------------------ revno: 2719.1.3 revision-id: knielsen(a)knielsen-hq.org-20090831112339-td6t7ljy4w2oamvt parent: knielsen(a)knielsen-hq.org-20090829194804-rfq9dbai99do12ub parent: knielsen(a)knielsen-hq.org-20090831110744-ru07dzndci7cf92k committer: knielsen(a)knielsen-hq.org branch nick: work timestamp: Mon 2009-08-31 13:23:39 +0200 message: Merge Paul's maria-pbxt-rc2 branch into MariaDB. Includes disabling PBMS (per Paul's recommendation) and test file fixes. Signed-off-by: Kristian Nielsen <knielsen(a)knielsen-hq.org> removed: storage/pbxt/src/streaming_xt.cc streaming_xt.cc-20090326121724-x683v32twzr3fi0y-57 storage/pbxt/src/streaming_xt.h streaming_xt.h-20090326121724-x683v32twzr3fi0y-58 added: mysql-test/suite/pbxt/t/lowercase_table_grant-master.opt lowercase_table_gran-20090828130852-3duj73io0g345hxj-1 mysql-test/suite/pbxt/t/lowercase_table_qcache-master.opt lowercase_table_qcac-20090828130852-3duj73io0g345hxj-2 mysql-test/suite/pbxt/t/lowercase_view-master.opt lowercase_viewmaster-20090828130852-3duj73io0g345hxj-3 mysql-test/suite/pbxt/t/udf-master.opt udfmaster.opt-20090828130852-3duj73io0g345hxj-4 storage/pbxt/src/pbms_enabled.cc pbms_enabled.cc-20090818074502-tybcb62hp1kdrk3l-1 storage/pbxt/src/pbms_enabled.h pbms_enabled.h-20090818074502-tybcb62hp1kdrk3l-2 modified: mysql-test/suite/pbxt/r/alter_table.result alter_table.result-20090402100035-4ilk9i91sh65vjcb-5 mysql-test/suite/pbxt/r/analyze.result analyze.result-20090402100035-4ilk9i91sh65vjcb-7 mysql-test/suite/pbxt/r/auto_increment.result auto_increment.resul-20090402100035-4ilk9i91sh65vjcb-9 mysql-test/suite/pbxt/r/delete.result delete.result-20090402100035-4ilk9i91sh65vjcb-31 mysql-test/suite/pbxt/r/distinct.result distinct.result-20090402100035-4ilk9i91sh65vjcb-34 mysql-test/suite/pbxt/r/func_group.result func_group.result-20090402100035-4ilk9i91sh65vjcb-50 mysql-test/suite/pbxt/r/func_math.result func_math.result-20090402100035-4ilk9i91sh65vjcb-55 mysql-test/suite/pbxt/r/func_str.result func_str.result-20090402100035-4ilk9i91sh65vjcb-61 mysql-test/suite/pbxt/r/grant.result grant.result-20090402100035-4ilk9i91sh65vjcb-65 mysql-test/suite/pbxt/r/group_min_max.result group_min_max.result-20090402100035-4ilk9i91sh65vjcb-69 mysql-test/suite/pbxt/r/join.result join.result-20090402100035-4ilk9i91sh65vjcb-79 mysql-test/suite/pbxt/r/join_nested.result join_nested.result-20090402100035-4ilk9i91sh65vjcb-81 mysql-test/suite/pbxt/r/key.result key.result-20090402100035-4ilk9i91sh65vjcb-83 mysql-test/suite/pbxt/r/key_cache.result key_cache.result-20090402100035-4ilk9i91sh65vjcb-84 mysql-test/suite/pbxt/r/key_diff.result key_diff.result-20090402100035-4ilk9i91sh65vjcb-85 mysql-test/suite/pbxt/r/lowercase_view.result lowercase_view.resul-20090402100035-4ilk9i91sh65vjcb-96 mysql-test/suite/pbxt/r/mysqlshow.result mysqlshow.result-20090402100035-4ilk9i91sh65vjcb-101 mysql-test/suite/pbxt/r/null.result null.result-20090402100035-4ilk9i91sh65vjcb-104 mysql-test/suite/pbxt/r/null_key.result null_key.result-20090402100035-4ilk9i91sh65vjcb-105 mysql-test/suite/pbxt/r/partition_pruning.result partition_pruning.re-20090402100035-4ilk9i91sh65vjcb-117 mysql-test/suite/pbxt/r/pbxt_bugs.result pbxt_bugs.result-20090402100035-4ilk9i91sh65vjcb-120 mysql-test/suite/pbxt/r/pbxt_ref_int.result pbxt_ref_int.result-20090402100035-4ilk9i91sh65vjcb-123 mysql-test/suite/pbxt/r/preload.result preload.result-20090402100035-4ilk9i91sh65vjcb-127 mysql-test/suite/pbxt/r/ps_1general.result ps_1general.result-20090402100035-4ilk9i91sh65vjcb-130 mysql-test/suite/pbxt/r/range.result range.result-20090402100035-4ilk9i91sh65vjcb-132 mysql-test/suite/pbxt/r/schema.result schema.result-20090402100035-4ilk9i91sh65vjcb-138 mysql-test/suite/pbxt/r/select.result select.result-20090402100035-4ilk9i91sh65vjcb-139 mysql-test/suite/pbxt/r/select_safe.result select_safe.result-20090402100035-4ilk9i91sh65vjcb-141 mysql-test/suite/pbxt/r/subselect.result subselect.result-20090402100035-4ilk9i91sh65vjcb-146 mysql-test/suite/pbxt/r/type_enum.result type_enum.result-20090402100035-4ilk9i91sh65vjcb-155 mysql-test/suite/pbxt/r/type_ranges.result type_ranges.result-20090402100035-4ilk9i91sh65vjcb-159 mysql-test/suite/pbxt/r/type_timestamp.result type_timestamp.resul-20090402100035-4ilk9i91sh65vjcb-162 mysql-test/suite/pbxt/r/union.result union.result-20090402100035-4ilk9i91sh65vjcb-166 mysql-test/suite/pbxt/r/view_grant.result view_grant.result-20090402100035-4ilk9i91sh65vjcb-169 mysql-test/suite/pbxt/t/auto_increment.test auto_increment.test-20090402100035-4ilk9i91sh65vjcb-176 mysql-test/suite/pbxt/t/delete.test delete.test-20090402100035-4ilk9i91sh65vjcb-198 mysql-test/suite/pbxt/t/join_nested.test join_nested.test-20090402100035-4ilk9i91sh65vjcb-248 mysql-test/suite/pbxt/t/null.test null.test-20090402100035-4ilk9i91sh65vjcb-271 mysql-test/suite/pbxt/t/pbxt_bugs.test pbxt_bugs.test-20090402100035-4ilk9i91sh65vjcb-287 mysql-test/suite/pbxt/t/rename.test rename.test-20090402100035-4ilk9i91sh65vjcb-301 mysql-test/suite/pbxt/t/schema.test schema.test-20090402100035-4ilk9i91sh65vjcb-305 mysql-test/suite/pbxt/t/type_enum.test type_enum.test-20090402100035-4ilk9i91sh65vjcb-322 mysql-test/suite/pbxt/t/union.test union.test-20090402100035-4ilk9i91sh65vjcb-333 storage/pbxt/ChangeLog changelog-20090326121724-x683v32twzr3fi0y-3 storage/pbxt/src/Makefile.am makefile.am-20090326121724-x683v32twzr3fi0y-13 storage/pbxt/src/cache_xt.cc cache_xt.cc-20090326121724-x683v32twzr3fi0y-16 storage/pbxt/src/cache_xt.h cache_xt.h-20090326121724-x683v32twzr3fi0y-17 storage/pbxt/src/ccutils_xt.cc ccutils_xt.cc-20090326121724-x683v32twzr3fi0y-18 storage/pbxt/src/database_xt.cc database_xt.cc-20090326121724-x683v32twzr3fi0y-20 storage/pbxt/src/datadic_xt.cc datadic_xt.cc-20090326121724-x683v32twzr3fi0y-22 storage/pbxt/src/datadic_xt.h datadic_xt.h-20090326121724-x683v32twzr3fi0y-23 storage/pbxt/src/datalog_xt.cc datalog_xt.cc-20090326121724-x683v32twzr3fi0y-24 storage/pbxt/src/datalog_xt.h datalog_xt.h-20090326121724-x683v32twzr3fi0y-25 storage/pbxt/src/discover_xt.cc discover_xt.cc-20090326121724-x683v32twzr3fi0y-26 storage/pbxt/src/filesys_xt.cc filesys_xt.cc-20090326121724-x683v32twzr3fi0y-28 storage/pbxt/src/filesys_xt.h filesys_xt.h-20090326121724-x683v32twzr3fi0y-29 storage/pbxt/src/ha_pbxt.cc ha_pbxt.cc-20090326121724-x683v32twzr3fi0y-30 storage/pbxt/src/ha_pbxt.h ha_pbxt.h-20090326121724-x683v32twzr3fi0y-31 storage/pbxt/src/ha_xtsys.cc ha_xtsys.cc-20090326121724-x683v32twzr3fi0y-32 storage/pbxt/src/ha_xtsys.h ha_xtsys.h-20090326121724-x683v32twzr3fi0y-33 storage/pbxt/src/hashtab_xt.cc hashtab_xt.cc-20090326121724-x683v32twzr3fi0y-34 storage/pbxt/src/heap_xt.cc heap_xt.cc-20090326121724-x683v32twzr3fi0y-36 storage/pbxt/src/heap_xt.h heap_xt.h-20090326121724-x683v32twzr3fi0y-37 storage/pbxt/src/index_xt.cc index_xt.cc-20090326121724-x683v32twzr3fi0y-38 storage/pbxt/src/index_xt.h index_xt.h-20090326121724-x683v32twzr3fi0y-39 storage/pbxt/src/lock_xt.cc lock_xt.cc-20090326121724-x683v32twzr3fi0y-42 storage/pbxt/src/lock_xt.h lock_xt.h-20090326121724-x683v32twzr3fi0y-43 storage/pbxt/src/locklist_xt.cc locklist_xt.cc-20090326121724-x683v32twzr3fi0y-44 storage/pbxt/src/locklist_xt.h locklist_xt.h-20090326121724-x683v32twzr3fi0y-45 storage/pbxt/src/memory_xt.cc memory_xt.cc-20090326121724-x683v32twzr3fi0y-46 storage/pbxt/src/memory_xt.h memory_xt.h-20090326121724-x683v32twzr3fi0y-47 storage/pbxt/src/myxt_xt.cc myxt_xt.cc-20090326121724-x683v32twzr3fi0y-48 storage/pbxt/src/myxt_xt.h myxt_xt.h-20090326121724-x683v32twzr3fi0y-49 storage/pbxt/src/pbms.h pbms.h-20090326121724-x683v32twzr3fi0y-50 storage/pbxt/src/pthread_xt.cc pthread_xt.cc-20090326121724-x683v32twzr3fi0y-51 storage/pbxt/src/restart_xt.cc restart_xt.cc-20090326121724-x683v32twzr3fi0y-53 storage/pbxt/src/restart_xt.h restart_xt.h-20090326121724-x683v32twzr3fi0y-54 storage/pbxt/src/sortedlist_xt.cc sortedlist_xt.cc-20090326121724-x683v32twzr3fi0y-55 storage/pbxt/src/strutil_xt.cc strutil_xt.cc-20090326121724-x683v32twzr3fi0y-59 storage/pbxt/src/systab_xt.cc systab_xt.cc-20090326121724-x683v32twzr3fi0y-61 storage/pbxt/src/systab_xt.h systab_xt.h-20090326121724-x683v32twzr3fi0y-62 storage/pbxt/src/tabcache_xt.cc tabcache_xt.cc-20090326121724-x683v32twzr3fi0y-63 storage/pbxt/src/tabcache_xt.h tabcache_xt.h-20090326121724-x683v32twzr3fi0y-64 storage/pbxt/src/table_xt.cc table_xt.cc-20090326121724-x683v32twzr3fi0y-65 storage/pbxt/src/table_xt.h table_xt.h-20090326121724-x683v32twzr3fi0y-66 storage/pbxt/src/thread_xt.cc thread_xt.cc-20090326121724-x683v32twzr3fi0y-67 storage/pbxt/src/thread_xt.h thread_xt.h-20090326121724-x683v32twzr3fi0y-68 storage/pbxt/src/trace_xt.cc trace_xt.cc-20090326121724-x683v32twzr3fi0y-69 storage/pbxt/src/trace_xt.h trace_xt.h-20090326121724-x683v32twzr3fi0y-70 storage/pbxt/src/util_xt.cc util_xt.cc-20090326121724-x683v32twzr3fi0y-71 storage/pbxt/src/xaction_xt.cc xaction_xt.cc-20090326121724-x683v32twzr3fi0y-74 storage/pbxt/src/xaction_xt.h xaction_xt.h-20090326121724-x683v32twzr3fi0y-75 storage/pbxt/src/xactlog_xt.cc xactlog_xt.cc-20090326121724-x683v32twzr3fi0y-76 storage/pbxt/src/xactlog_xt.h xactlog_xt.h-20090326121724-x683v32twzr3fi0y-77 storage/pbxt/src/xt_config.h xt_config.h-20090326121724-x683v32twzr3fi0y-78 storage/pbxt/src/xt_defs.h xt_defs.h-20090326121724-x683v32twzr3fi0y-79 ------------------------------------------------------------ revno: 2719.2.4 revision-id: knielsen(a)knielsen-hq.org-20090831110744-ru07dzndci7cf92k parent: paul.mccullagh(a)primebase.org-20090818074653-8mbinxoo0ylqqqr9 committer: knielsen(a)knielsen-hq.org branch nick: maria-pbxt-rc2 timestamp: Mon 2009-08-31 13:07:44 +0200 message: PBXT merge fixes. - Disable PBMS in MariaDB (as per recommendation of Paul). - Add missing *-master.opt files to make testsuite work on case-sensitive file systems. - Result file updates. added: mysql-test/suite/pbxt/t/lowercase_table_grant-master.opt lowercase_table_gran-20090828130852-3duj73io0g345hxj-1 mysql-test/suite/pbxt/t/lowercase_table_qcache-master.opt lowercase_table_qcac-20090828130852-3duj73io0g345hxj-2 mysql-test/suite/pbxt/t/lowercase_view-master.opt lowercase_viewmaster-20090828130852-3duj73io0g345hxj-3 mysql-test/suite/pbxt/t/udf-master.opt udfmaster.opt-20090828130852-3duj73io0g345hxj-4 modified: mysql-test/suite/pbxt/r/lowercase_view.result lowercase_view.resul-20090402100035-4ilk9i91sh65vjcb-96 mysql-test/suite/pbxt/r/ps_1general.result ps_1general.result-20090402100035-4ilk9i91sh65vjcb-130 storage/pbxt/src/Makefile.am makefile.am-20090326121724-x683v32twzr3fi0y-13 storage/pbxt/src/xt_config.h xt_config.h-20090326121724-x683v32twzr3fi0y-78 ------------------------------------------------------------ revno: 2719.2.3 revision-id: paul.mccullagh(a)primebase.org-20090818074653-8mbinxoo0ylqqqr9 parent: paul.mccullagh(a)primebase.org-20090817155758-zliin5rbosn30az8 committer: Paul McCullagh <paul.mccullagh(a)primebase.org> branch nick: trunk timestamp: Tue 2009-08-18 09:46:53 +0200 message: Merged changes for bug fix update 1.0.08c RC2 removed: storage/pbxt/src/streaming_xt.cc streaming_xt.cc-20090326121724-x683v32twzr3fi0y-57 storage/pbxt/src/streaming_xt.h streaming_xt.h-20090326121724-x683v32twzr3fi0y-58 added: storage/pbxt/src/pbms_enabled.cc pbms_enabled.cc-20090818074502-tybcb62hp1kdrk3l-1 storage/pbxt/src/pbms_enabled.h pbms_enabled.h-20090818074502-tybcb62hp1kdrk3l-2 modified: storage/pbxt/ChangeLog changelog-20090326121724-x683v32twzr3fi0y-3 storage/pbxt/src/datadic_xt.cc datadic_xt.cc-20090326121724-x683v32twzr3fi0y-22 storage/pbxt/src/datadic_xt.h datadic_xt.h-20090326121724-x683v32twzr3fi0y-23 storage/pbxt/src/datalog_xt.cc datalog_xt.cc-20090326121724-x683v32twzr3fi0y-24 storage/pbxt/src/datalog_xt.h datalog_xt.h-20090326121724-x683v32twzr3fi0y-25 storage/pbxt/src/filesys_xt.cc filesys_xt.cc-20090326121724-x683v32twzr3fi0y-28 storage/pbxt/src/ha_pbxt.cc ha_pbxt.cc-20090326121724-x683v32twzr3fi0y-30 storage/pbxt/src/index_xt.cc index_xt.cc-20090326121724-x683v32twzr3fi0y-38 storage/pbxt/src/myxt_xt.cc myxt_xt.cc-20090326121724-x683v32twzr3fi0y-48 storage/pbxt/src/myxt_xt.h myxt_xt.h-20090326121724-x683v32twzr3fi0y-49 storage/pbxt/src/pbms.h pbms.h-20090326121724-x683v32twzr3fi0y-50 storage/pbxt/src/pthread_xt.cc pthread_xt.cc-20090326121724-x683v32twzr3fi0y-51 storage/pbxt/src/restart_xt.cc restart_xt.cc-20090326121724-x683v32twzr3fi0y-53 storage/pbxt/src/strutil_xt.cc strutil_xt.cc-20090326121724-x683v32twzr3fi0y-59 storage/pbxt/src/table_xt.cc table_xt.cc-20090326121724-x683v32twzr3fi0y-65 storage/pbxt/src/table_xt.h table_xt.h-20090326121724-x683v32twzr3fi0y-66 storage/pbxt/src/xactlog_xt.cc xactlog_xt.cc-20090326121724-x683v32twzr3fi0y-76 storage/pbxt/src/xt_config.h xt_config.h-20090326121724-x683v32twzr3fi0y-78 ------------------------------------------------------------ revno: 2719.2.2 revision-id: paul.mccullagh(a)primebase.org-20090817155758-zliin5rbosn30az8 parent: paul.mccullagh(a)primebase.org-20090817111236-1pgxzp5x9l74q0ug committer: Paul McCullagh <paul.mccullagh(a)primebase.org> branch nick: trunk timestamp: Mon 2009-08-17 17:57:58 +0200 message: Updated all tests for RC2 modified: mysql-test/suite/pbxt/r/analyze.result analyze.result-20090402100035-4ilk9i91sh65vjcb-7 mysql-test/suite/pbxt/r/auto_increment.result auto_increment.resul-20090402100035-4ilk9i91sh65vjcb-9 mysql-test/suite/pbxt/r/delete.result delete.result-20090402100035-4ilk9i91sh65vjcb-31 mysql-test/suite/pbxt/r/distinct.result distinct.result-20090402100035-4ilk9i91sh65vjcb-34 mysql-test/suite/pbxt/r/func_group.result func_group.result-20090402100035-4ilk9i91sh65vjcb-50 mysql-test/suite/pbxt/r/func_math.result func_math.result-20090402100035-4ilk9i91sh65vjcb-55 mysql-test/suite/pbxt/r/func_str.result func_str.result-20090402100035-4ilk9i91sh65vjcb-61 mysql-test/suite/pbxt/r/grant.result grant.result-20090402100035-4ilk9i91sh65vjcb-65 mysql-test/suite/pbxt/r/group_min_max.result group_min_max.result-20090402100035-4ilk9i91sh65vjcb-69 mysql-test/suite/pbxt/r/join.result join.result-20090402100035-4ilk9i91sh65vjcb-79 mysql-test/suite/pbxt/r/join_nested.result join_nested.result-20090402100035-4ilk9i91sh65vjcb-81 mysql-test/suite/pbxt/r/key.result key.result-20090402100035-4ilk9i91sh65vjcb-83 mysql-test/suite/pbxt/r/key_cache.result key_cache.result-20090402100035-4ilk9i91sh65vjcb-84 mysql-test/suite/pbxt/r/key_diff.result key_diff.result-20090402100035-4ilk9i91sh65vjcb-85 mysql-test/suite/pbxt/r/mysqlshow.result mysqlshow.result-20090402100035-4ilk9i91sh65vjcb-101 mysql-test/suite/pbxt/r/null.result null.result-20090402100035-4ilk9i91sh65vjcb-104 mysql-test/suite/pbxt/r/null_key.result null_key.result-20090402100035-4ilk9i91sh65vjcb-105 mysql-test/suite/pbxt/r/partition_pruning.result partition_pruning.re-20090402100035-4ilk9i91sh65vjcb-117 mysql-test/suite/pbxt/r/pbxt_bugs.result pbxt_bugs.result-20090402100035-4ilk9i91sh65vjcb-120 mysql-test/suite/pbxt/r/pbxt_ref_int.result pbxt_ref_int.result-20090402100035-4ilk9i91sh65vjcb-123 mysql-test/suite/pbxt/r/preload.result preload.result-20090402100035-4ilk9i91sh65vjcb-127 mysql-test/suite/pbxt/r/ps_1general.result ps_1general.result-20090402100035-4ilk9i91sh65vjcb-130 mysql-test/suite/pbxt/r/range.result range.result-20090402100035-4ilk9i91sh65vjcb-132 mysql-test/suite/pbxt/r/schema.result schema.result-20090402100035-4ilk9i91sh65vjcb-138 mysql-test/suite/pbxt/r/select.result select.result-20090402100035-4ilk9i91sh65vjcb-139 mysql-test/suite/pbxt/r/select_safe.result select_safe.result-20090402100035-4ilk9i91sh65vjcb-141 mysql-test/suite/pbxt/r/subselect.result subselect.result-20090402100035-4ilk9i91sh65vjcb-146 mysql-test/suite/pbxt/r/type_enum.result type_enum.result-20090402100035-4ilk9i91sh65vjcb-155 mysql-test/suite/pbxt/r/type_ranges.result type_ranges.result-20090402100035-4ilk9i91sh65vjcb-159 mysql-test/suite/pbxt/r/type_timestamp.result type_timestamp.resul-20090402100035-4ilk9i91sh65vjcb-162 mysql-test/suite/pbxt/r/union.result union.result-20090402100035-4ilk9i91sh65vjcb-166 mysql-test/suite/pbxt/r/view_grant.result view_grant.result-20090402100035-4ilk9i91sh65vjcb-169 mysql-test/suite/pbxt/t/auto_increment.test auto_increment.test-20090402100035-4ilk9i91sh65vjcb-176 mysql-test/suite/pbxt/t/delete.test delete.test-20090402100035-4ilk9i91sh65vjcb-198 mysql-test/suite/pbxt/t/join_nested.test join_nested.test-20090402100035-4ilk9i91sh65vjcb-248 mysql-test/suite/pbxt/t/null.test null.test-20090402100035-4ilk9i91sh65vjcb-271 mysql-test/suite/pbxt/t/pbxt_bugs.test pbxt_bugs.test-20090402100035-4ilk9i91sh65vjcb-287 mysql-test/suite/pbxt/t/rename.test rename.test-20090402100035-4ilk9i91sh65vjcb-301 mysql-test/suite/pbxt/t/schema.test schema.test-20090402100035-4ilk9i91sh65vjcb-305 mysql-test/suite/pbxt/t/type_enum.test type_enum.test-20090402100035-4ilk9i91sh65vjcb-322 mysql-test/suite/pbxt/t/union.test union.test-20090402100035-4ilk9i91sh65vjcb-333 ------------------------------------------------------------ revno: 2719.2.1 revision-id: paul.mccullagh(a)primebase.org-20090817111236-1pgxzp5x9l74q0ug parent: knielsen(a)knielsen-hq.org-20090805072137-wg97dcem1cxnzt3p committer: Paul McCullagh <paul.mccullagh(a)primebase.org> branch nick: trunk timestamp: Mon 2009-08-17 13:12:36 +0200 message: Updated PBXT to version 1.0.08 RC2 modified: mysql-test/suite/pbxt/r/alter_table.result alter_table.result-20090402100035-4ilk9i91sh65vjcb-5 storage/pbxt/ChangeLog changelog-20090326121724-x683v32twzr3fi0y-3 storage/pbxt/src/cache_xt.cc cache_xt.cc-20090326121724-x683v32twzr3fi0y-16 storage/pbxt/src/cache_xt.h cache_xt.h-20090326121724-x683v32twzr3fi0y-17 storage/pbxt/src/ccutils_xt.cc ccutils_xt.cc-20090326121724-x683v32twzr3fi0y-18 storage/pbxt/src/database_xt.cc database_xt.cc-20090326121724-x683v32twzr3fi0y-20 storage/pbxt/src/datadic_xt.cc datadic_xt.cc-20090326121724-x683v32twzr3fi0y-22 storage/pbxt/src/datadic_xt.h datadic_xt.h-20090326121724-x683v32twzr3fi0y-23 storage/pbxt/src/datalog_xt.cc datalog_xt.cc-20090326121724-x683v32twzr3fi0y-24 storage/pbxt/src/discover_xt.cc discover_xt.cc-20090326121724-x683v32twzr3fi0y-26 storage/pbxt/src/filesys_xt.cc filesys_xt.cc-20090326121724-x683v32twzr3fi0y-28 storage/pbxt/src/filesys_xt.h filesys_xt.h-20090326121724-x683v32twzr3fi0y-29 storage/pbxt/src/ha_pbxt.cc ha_pbxt.cc-20090326121724-x683v32twzr3fi0y-30 storage/pbxt/src/ha_pbxt.h ha_pbxt.h-20090326121724-x683v32twzr3fi0y-31 storage/pbxt/src/ha_xtsys.cc ha_xtsys.cc-20090326121724-x683v32twzr3fi0y-32 storage/pbxt/src/ha_xtsys.h ha_xtsys.h-20090326121724-x683v32twzr3fi0y-33 storage/pbxt/src/hashtab_xt.cc hashtab_xt.cc-20090326121724-x683v32twzr3fi0y-34 storage/pbxt/src/heap_xt.cc heap_xt.cc-20090326121724-x683v32twzr3fi0y-36 storage/pbxt/src/heap_xt.h heap_xt.h-20090326121724-x683v32twzr3fi0y-37 storage/pbxt/src/index_xt.cc index_xt.cc-20090326121724-x683v32twzr3fi0y-38 storage/pbxt/src/index_xt.h index_xt.h-20090326121724-x683v32twzr3fi0y-39 storage/pbxt/src/lock_xt.cc lock_xt.cc-20090326121724-x683v32twzr3fi0y-42 storage/pbxt/src/lock_xt.h lock_xt.h-20090326121724-x683v32twzr3fi0y-43 storage/pbxt/src/locklist_xt.cc locklist_xt.cc-20090326121724-x683v32twzr3fi0y-44 storage/pbxt/src/locklist_xt.h locklist_xt.h-20090326121724-x683v32twzr3fi0y-45 storage/pbxt/src/memory_xt.cc memory_xt.cc-20090326121724-x683v32twzr3fi0y-46 storage/pbxt/src/memory_xt.h memory_xt.h-20090326121724-x683v32twzr3fi0y-47 storage/pbxt/src/myxt_xt.cc myxt_xt.cc-20090326121724-x683v32twzr3fi0y-48 storage/pbxt/src/myxt_xt.h myxt_xt.h-20090326121724-x683v32twzr3fi0y-49 storage/pbxt/src/pbms.h pbms.h-20090326121724-x683v32twzr3fi0y-50 storage/pbxt/src/restart_xt.cc restart_xt.cc-20090326121724-x683v32twzr3fi0y-53 storage/pbxt/src/restart_xt.h restart_xt.h-20090326121724-x683v32twzr3fi0y-54 storage/pbxt/src/sortedlist_xt.cc sortedlist_xt.cc-20090326121724-x683v32twzr3fi0y-55 storage/pbxt/src/streaming_xt.cc streaming_xt.cc-20090326121724-x683v32twzr3fi0y-57 storage/pbxt/src/strutil_xt.cc strutil_xt.cc-20090326121724-x683v32twzr3fi0y-59 storage/pbxt/src/systab_xt.cc systab_xt.cc-20090326121724-x683v32twzr3fi0y-61 storage/pbxt/src/systab_xt.h systab_xt.h-20090326121724-x683v32twzr3fi0y-62 storage/pbxt/src/tabcache_xt.cc tabcache_xt.cc-20090326121724-x683v32twzr3fi0y-63 storage/pbxt/src/tabcache_xt.h tabcache_xt.h-20090326121724-x683v32twzr3fi0y-64 storage/pbxt/src/table_xt.cc table_xt.cc-20090326121724-x683v32twzr3fi0y-65 storage/pbxt/src/table_xt.h table_xt.h-20090326121724-x683v32twzr3fi0y-66 storage/pbxt/src/thread_xt.cc thread_xt.cc-20090326121724-x683v32twzr3fi0y-67 storage/pbxt/src/thread_xt.h thread_xt.h-20090326121724-x683v32twzr3fi0y-68 storage/pbxt/src/trace_xt.cc trace_xt.cc-20090326121724-x683v32twzr3fi0y-69 storage/pbxt/src/trace_xt.h trace_xt.h-20090326121724-x683v32twzr3fi0y-70 storage/pbxt/src/util_xt.cc util_xt.cc-20090326121724-x683v32twzr3fi0y-71 storage/pbxt/src/xaction_xt.cc xaction_xt.cc-20090326121724-x683v32twzr3fi0y-74 storage/pbxt/src/xaction_xt.h xaction_xt.h-20090326121724-x683v32twzr3fi0y-75 storage/pbxt/src/xactlog_xt.cc xactlog_xt.cc-20090326121724-x683v32twzr3fi0y-76 storage/pbxt/src/xactlog_xt.h xactlog_xt.h-20090326121724-x683v32twzr3fi0y-77 storage/pbxt/src/xt_config.h xt_config.h-20090326121724-x683v32twzr3fi0y-78 storage/pbxt/src/xt_defs.h xt_defs.h-20090326121724-x683v32twzr3fi0y-79 ------------------------------------------------------------ revno: 2719.1.2 revision-id: knielsen(a)knielsen-hq.org-20090829194804-rfq9dbai99do12ub parent: knielsen(a)knielsen-hq.org-20090829190446-bm072wpuvueu2o59 committer: knielsen(a)knielsen-hq.org branch nick: mariadb-5.1 timestamp: Sat 2009-08-29 21:48:04 +0200 message: Add missign header file to `make dist`. modified: storage/xtradb/Makefile.am makefile.am-20081201061010-zymrrwrczns2vrex-4 ------------------------------------------------------------ revno: 2719.1.1 revision-id: knielsen(a)knielsen-hq.org-20090829190446-bm072wpuvueu2o59 parent: knielsen(a)knielsen-hq.org-20090805072137-wg97dcem1cxnzt3p committer: knielsen(a)knielsen-hq.org branch nick: 5.1 timestamp: Sat 2009-08-29 21:04:46 +0200 message: Fix `make dist` after solaris build file fixes. Solaris build scripts under BUILD/ were changed, but Makefile.am was not updated to reflect the new set of file names. modified: BUILD/Makefile.am sp1f-makefile.am-20020102192940-dza66ux2yxyklupzjz4q3km3hvye5rnj Diff too large for email (15664 lines, the limit is 1000).

1 0

[Maria-developers] Rev 2750: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r11/
by Sergey Petrunya 01 Sep '09

01 Sep '09

At file:///home/psergey/dev/maria-5.1-table-elim-r11/ ------------------------------------------------------------ revno: 2750 revision-id: psergey(a)askmonty.org-20090901214116-n5bzyqlhgl0lamno parent: psergey(a)askmonty.org-20090831200209-7gbnkr7omjbgld1x committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Wed 2009-09-02 01:41:16 +0400 message: MWL#17: Table elimination - Address review feedback R4: better comments, formatting === modified file 'sql-bench/test-table-elimination.sh' --- a/sql-bench/test-table-elimination.sh 2009-06-23 20:06:13 +0000 +++ b/sql-bench/test-table-elimination.sh 2009-09-01 21:41:16 +0000 @@ -290,10 +290,6 @@ timestr(timediff($end_time, $loop_time),"all") . "\n"; -### -### TODO... -### - ; #### === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-31 20:02:09 +0000 +++ b/sql/opt_table_elimination.cc 2009-09-01 21:41:16 +0000 @@ -24,8 +24,10 @@ elimination is as follows: suppose we have a left join SELECT * FROM t1 LEFT JOIN - (t2 JOIN t3) ON t3.primary_key=t1.col AND - t4.primary_key=t2.col + (t2 JOIN t3) ON t2.primary_key=t1.col AND + t2.primary_key=t2.col + WHERE ... + such that * columns of the inner tables are not used anywhere ouside the outer join (not in WHERE, not in GROUP/ORDER BY clause, not in select list etc etc), @@ -41,11 +43,11 @@ MODULE INTERFACE ================ - The module has one entry point - eliminate_tables() function, which one - needs to call (once) at some point before the join optimization. + The module has one entry point - the eliminate_tables() function, which one + needs to call (once) at some point before join optimization. eliminate_tables() operates over the JOIN structures. Logically, it - removes the right sides of outer join nests. Physically, it changes the - following members: + removes the inner tables of an outer join operation together with the + operation itself. Physically, it changes the following members: * Eliminated tables are marked as constant and moved to the front of the join order. @@ -119,13 +121,14 @@ = No outgoing edges. Once we reach it, we know we can eliminate the outer join. A module may depend on multiple values, and hence its primary attribute is - the number of its depedencies that are not bound. + the number of its arguments that are not bound. The algorithm starts with equality nodes that don't have any incoming edges (their expressions are either constant or depend only on tables that are outside of the outer join in question) and performns a breadth-first traversal. If we reach the outer join nest node, it means outer join is - functionally dependent and can be eliminated. Otherwise it cannot be. + functionally dependent and can be eliminated. Otherwise it cannot be + eliminated. HANDLING MULTIPLE NESTED OUTER JOINS ==================================== @@ -230,8 +233,9 @@ Field *field; /* Field this object is representing */ /* Iteration over unbound modules that are our dependencies */ - virtual Iterator init_unbound_modules_iter(char *buf); - virtual Dep_module* get_next_unbound_module(Dep_analysis_context *dac, Iterator iter); + Iterator init_unbound_modules_iter(char *buf); + Dep_module* get_next_unbound_module(Dep_analysis_context *dac, + Iterator iter); void make_unbound_modules_iter_skip_keys(Iterator iter); @@ -269,9 +273,11 @@ /* A table value. There is one Dep_value_table object for every table that can potentially be eliminated. - Dependencies: - - table depends on any of its unique keys - - has its fields and embedding outer join as dependency + + Table becomes bound as soon as some of its unique keys becomes bound + Once the table is bound: + - all of its fields are bound + - its embedding outer join has one less unknown argument */ class Dep_value_table : public Dep_value @@ -280,14 +286,15 @@ Dep_value_table(TABLE *table_arg) : table(table_arg), fields(NULL), keys(NULL) {} - TABLE *table; - Dep_value_field *fields; /* Ordered list of fields that belong to this table */ + TABLE *table; /* Table this object is representing */ + /* Ordered list of fields that belong to this table */ + Dep_value_field *fields; Dep_module_key *keys; /* Ordered list of Unique keys in this table */ /* Iteration over unbound modules that are our dependencies */ Iterator init_unbound_modules_iter(char *buf); - Dep_module* get_next_unbound_module(Dep_analysis_context *dac, Iterator iter); - + Dep_module* get_next_unbound_module(Dep_analysis_context *dac, + Iterator iter); static const size_t iterator_size; private: class Module_iter @@ -295,7 +302,7 @@ public: /* Space for field iterator */ char buf[Dep_value_field::iterator_size]; - /* !NULL <=> iterating over dependenant modules of this field */ + /* !NULL <=> iterating over depdenent modules of this field */ Dep_value_field *field_dep; bool returned_goal; }; @@ -339,7 +346,8 @@ /* Iteration over values that */ typedef char *Iterator; virtual Iterator init_unbound_values_iter(char *buf)=0; - virtual Dep_value* get_next_unbound_value(Dep_analysis_context *dac, Iterator iter)=0; + virtual Dep_value* get_next_unbound_value(Dep_analysis_context *dac, + Iterator iter)=0; static const size_t iterator_size; protected: uint unbound_args; @@ -385,9 +393,9 @@ /* - A Unique key - - Unique key depends on all of its components - - Key's table is its dependency + A Unique key module + - Unique key has all of its components as arguments + - Once unique key is bound, its table value is known */ class Dep_module_key: public Dep_module @@ -539,8 +547,8 @@ The idea behind table elimination is that if we have an outer join: SELECT * FROM t1 LEFT JOIN - (t2 JOIN t3) ON t3.primary_key=t1.col AND - t4.primary_key=t2.col + (t2 JOIN t3) ON t2.primary_key=t1.col AND + t3.primary_key=t2.col such that 1. columns of the inner tables are not used anywhere ouside the outer @@ -559,8 +567,8 @@ tables that are not used in select list/GROUP BY/ORDER BY/HAVING/etc and thus can possibly be eliminated. - After this, if #1 is met, the function calls eliminate_tables_for_list() - that checks #2. + After this, if #1 is met, the function calls eliminate_tables_for_list() + that checks #2. SIDE EFFECTS See the OVERVIEW section at the top of this file. @@ -656,7 +664,14 @@ select list, HAVING, other ON expressions, etc). DESCRIPTION - Perform table elimination in a given join list. + Perform table elimination in a given join list: + - First, walk through join list members and try doing table elimination for + them. + - Then, if the join list itself is an inner side of outer join + (on_expr!=NULL), then try to eliminate the entire join list. + + See "HANDLING MULTIPLE NESTED OUTER JOINS" section at the top of this file + for more detailed description and justification. RETURN TRUE The entire join list eliminated @@ -1677,7 +1692,8 @@ } -void Dep_value_field::make_unbound_modules_iter_skip_keys(Dep_value::Iterator iter) +void +Dep_value_field::make_unbound_modules_iter_skip_keys(Dep_value::Iterator iter) { ((Module_iter*)iter)->key_dep= NULL; }

1 0

[Maria-developers] [Merge] lp:~paul-mccullagh/maria/maria-pbxt-rc2 into lp:maria
by Paul McCullagh 01 Sep '09

01 Sep '09

Paul McCullagh has proposed merging lp:~paul-mccullagh/maria/maria-pbxt-rc2 into lp:maria. Requested reviews: Maria-captains (maria-captains) Updated the PBXT engine to RC2 (lp:pbxt/rc2), version 1.0.08c. All tests in the PBXT suite pass. Note that this branch does not yet include the entire history of PBXT because attempts to do this have failed so far due to problems with bzr. -- https://code.launchpad.net/~paul-mccullagh/maria/maria-pbxt-rc2/+merge/10303 Your team Maria developers is subscribed to branch lp:maria.

3 7

[Maria-developers] Progress (by Sanja): Avoid flushing keycache to disk in case of DROPping table (53)
by worklog-noreply＠askmonty.org 01 Sep '09

01 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Avoid flushing keycache to disk in case of DROPping table CREATION DATE..: Thu, 27 Aug 2009, 11:12 SUPERVISOR.....: Monty IMPLEMENTOR....: Sanja COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 53 (http://askmonty.org/worklog/?tid=53) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 5 ESTIMATE.......: 19 (hours remain) ORIG. ESTIMATE.: 24 PROGRESS NOTES: -=-=(Sanja - Tue, 01 Sep 2009, 15:20)=-=- moving to 5.0 part1 Worked 2 hours and estimate 19 hours remain (original estimate unchanged). -=-=(Sanja - Tue, 01 Sep 2009, 15:19)=-=- first patch Worked 3 hours and estimate 21 hours remain (original estimate unchanged). -=-=(Sanja - Thu, 27 Aug 2009, 11:15)=-=- High-Level Specification modified. --- /tmp/wklog.53.old.23490 2009-08-27 11:15:24.000000000 +0300 +++ /tmp/wklog.53.new.23490 2009-08-27 11:15:24.000000000 +0300 @@ -1 +1,3 @@ - +To make special handler call 'prepare to drop' which mark table as changed (in +case of server crash just before removing table it force table to be repaired) +then flush keycache without writing on disk. DESCRIPTION: When DROP TABLE executed keycache flushed to disk before removing table which is wasting of resources. So better to have table removed from keycache without writing changes on disk. HIGH-LEVEL SPECIFICATION: To make special handler call 'prepare to drop' which mark table as changed (in case of server crash just before removing table it force table to be repaired) then flush keycache without writing on disk. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Progress (by Sanja): Avoid flushing keycache to disk in case of DROPping table (53)
by worklog-noreply＠askmonty.org 01 Sep '09

01 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Avoid flushing keycache to disk in case of DROPping table CREATION DATE..: Thu, 27 Aug 2009, 11:12 SUPERVISOR.....: Monty IMPLEMENTOR....: Sanja COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 53 (http://askmonty.org/worklog/?tid=53) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 5 ESTIMATE.......: 19 (hours remain) ORIG. ESTIMATE.: 24 PROGRESS NOTES: -=-=(Sanja - Tue, 01 Sep 2009, 15:20)=-=- moving to 5.0 part1 Worked 2 hours and estimate 19 hours remain (original estimate unchanged). -=-=(Sanja - Tue, 01 Sep 2009, 15:19)=-=- first patch Worked 3 hours and estimate 21 hours remain (original estimate unchanged). -=-=(Sanja - Thu, 27 Aug 2009, 11:15)=-=- High-Level Specification modified. --- /tmp/wklog.53.old.23490 2009-08-27 11:15:24.000000000 +0300 +++ /tmp/wklog.53.new.23490 2009-08-27 11:15:24.000000000 +0300 @@ -1 +1,3 @@ - +To make special handler call 'prepare to drop' which mark table as changed (in +case of server crash just before removing table it force table to be repaired) +then flush keycache without writing on disk. DESCRIPTION: When DROP TABLE executed keycache flushed to disk before removing table which is wasting of resources. So better to have table removed from keycache without writing changes on disk. HIGH-LEVEL SPECIFICATION: To make special handler call 'prepare to drop' which mark table as changed (in case of server crash just before removing table it force table to be repaired) then flush keycache without writing on disk. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Progress (by Sanja): Avoid flushing keycache to disk in case of DROPping table (53)
by worklog-noreply＠askmonty.org 01 Sep '09

01 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Avoid flushing keycache to disk in case of DROPping table CREATION DATE..: Thu, 27 Aug 2009, 11:12 SUPERVISOR.....: Monty IMPLEMENTOR....: Sanja COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 53 (http://askmonty.org/worklog/?tid=53) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 3 ESTIMATE.......: 21 (hours remain) ORIG. ESTIMATE.: 24 PROGRESS NOTES: -=-=(Sanja - Tue, 01 Sep 2009, 15:19)=-=- first patch Worked 3 hours and estimate 21 hours remain (original estimate unchanged). -=-=(Sanja - Thu, 27 Aug 2009, 11:15)=-=- High-Level Specification modified. --- /tmp/wklog.53.old.23490 2009-08-27 11:15:24.000000000 +0300 +++ /tmp/wklog.53.new.23490 2009-08-27 11:15:24.000000000 +0300 @@ -1 +1,3 @@ - +To make special handler call 'prepare to drop' which mark table as changed (in +case of server crash just before removing table it force table to be repaired) +then flush keycache without writing on disk. DESCRIPTION: When DROP TABLE executed keycache flushed to disk before removing table which is wasting of resources. So better to have table removed from keycache without writing changes on disk. HIGH-LEVEL SPECIFICATION: To make special handler call 'prepare to drop' which mark table as changed (in case of server crash just before removing table it force table to be repaired) then flush keycache without writing on disk. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Progress (by Sanja): Avoid flushing keycache to disk in case of DROPping table (53)
by worklog-noreply＠askmonty.org 01 Sep '09

01 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Avoid flushing keycache to disk in case of DROPping table CREATION DATE..: Thu, 27 Aug 2009, 11:12 SUPERVISOR.....: Monty IMPLEMENTOR....: Sanja COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 53 (http://askmonty.org/worklog/?tid=53) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 3 ESTIMATE.......: 21 (hours remain) ORIG. ESTIMATE.: 24 PROGRESS NOTES: -=-=(Sanja - Tue, 01 Sep 2009, 15:19)=-=- first patch Worked 3 hours and estimate 21 hours remain (original estimate unchanged). -=-=(Sanja - Thu, 27 Aug 2009, 11:15)=-=- High-Level Specification modified. --- /tmp/wklog.53.old.23490 2009-08-27 11:15:24.000000000 +0300 +++ /tmp/wklog.53.new.23490 2009-08-27 11:15:24.000000000 +0300 @@ -1 +1,3 @@ - +To make special handler call 'prepare to drop' which mark table as changed (in +case of server crash just before removing table it force table to be repaired) +then flush keycache without writing on disk. DESCRIPTION: When DROP TABLE executed keycache flushed to disk before removing table which is wasting of resources. So better to have table removed from keycache without writing changes on disk. HIGH-LEVEL SPECIFICATION: To make special handler call 'prepare to drop' which mark table as changed (in case of server crash just before removing table it force table to be repaired) then flush keycache without writing on disk. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Progress (by Sanja): Backport key cache to 5.0 (52)
by worklog-noreply＠askmonty.org 01 Sep '09

01 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Backport key cache to 5.0 CREATION DATE..: Thu, 27 Aug 2009, 09:54 SUPERVISOR.....: Monty IMPLEMENTOR....: Monty COPIES TO......: Sanja CATEGORY.......: Server-Sprint TASK ID........: 52 (http://askmonty.org/worklog/?tid=52) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 9 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 8 PROGRESS NOTES: -=-=(Sanja - Tue, 01 Sep 2009, 15:18)=-=- testing Monty's patch Worked 3 hours and estimate 0 hours remain (original estimate increased by 1 hour). -=-=(Guest - Fri, 28 Aug 2009, 10:55)=-=- Low Level Design modified. --- /tmp/wklog.52.old.16945 2009-08-28 10:55:40.000000000 +0300 +++ /tmp/wklog.52.new.16945 2009-08-28 10:55:40.000000000 +0300 @@ -1 +1,22 @@ -Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. +Here is instructions of how to repeat this if needed: + +- Copy the following files from the latest MySQL 5.1 to MySQL 5.0: + +include/keycache.h +include/my_bit.h +mysys/mf_keycache.c +mysys/mf_keycaches.c +mysys/my_bit.c + +Apply the following change in mysys/mf_keycaches.c +81c81 +< static uchar *safe_hash_entry_get(SAFE_HASH_ENTRY *entry, size_t *length, +--- +> static uchar *safe_hash_entry_get(SAFE_HASH_ENTRY *entry, uint *length, + +recompile everything. + +Simple test to verify that this is working: +cd mysql-test +mysql-test-run --valgrind key_cache +(the --valgrind argument can be left out if you don't have valgrind installed) -=-=(Guest - Thu, 27 Aug 2009, 21:52)=-=- - Planning code (Email dicussions & discussions with Sanja): 2 hours. - Backport of code (2 hours). Notice that mysys/mf_keycaches.c::safe_hash_entry_get() is different from the code in MySQL 5.1 - Testing code with mysql-test-run (especially running the key_cache.test with valgrind) - Tested the server with sql-bench Worked 6 hours and estimate 2 hours remain (original estimate unchanged). -=-=(Sanja - Thu, 27 Aug 2009, 09:56)=-=- High-Level Specification modified. --- /tmp/wklog.52.old.20094 2009-08-27 09:56:05.000000000 +0300 +++ /tmp/wklog.52.new.20094 2009-08-27 09:56:05.000000000 +0300 @@ -1 +1 @@ - +Backport keycache to 5.0 -=-=(Sanja - Thu, 27 Aug 2009, 09:55)=-=- Low Level Design modified. --- /tmp/wklog.52.old.20035 2009-08-27 09:55:25.000000000 +0300 +++ /tmp/wklog.52.new.20035 2009-08-27 09:55:25.000000000 +0300 @@ -1 +1 @@ - +Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. DESCRIPTION: Backport keycache to bring bugfixes made in 5.1. HIGH-LEVEL SPECIFICATION: Backport keycache to 5.0 LOW-LEVEL DESIGN: Here is instructions of how to repeat this if needed: - Copy the following files from the latest MySQL 5.1 to MySQL 5.0: include/keycache.h include/my_bit.h mysys/mf_keycache.c mysys/mf_keycaches.c mysys/my_bit.c Apply the following change in mysys/mf_keycaches.c 81c81 < static uchar *safe_hash_entry_get(SAFE_HASH_ENTRY *entry, size_t *length, --- > static uchar *safe_hash_entry_get(SAFE_HASH_ENTRY *entry, uint *length, recompile everything. Simple test to verify that this is working: cd mysql-test mysql-test-run --valgrind key_cache (the --valgrind argument can be left out if you don't have valgrind installed) ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Progress (by Sanja): Backport key cache to 5.0 (52)
by worklog-noreply＠askmonty.org 01 Sep '09

01 Sep '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Backport key cache to 5.0 CREATION DATE..: Thu, 27 Aug 2009, 09:54 SUPERVISOR.....: Monty IMPLEMENTOR....: Monty COPIES TO......: Sanja CATEGORY.......: Server-Sprint TASK ID........: 52 (http://askmonty.org/worklog/?tid=52) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 9 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 8 PROGRESS NOTES: -=-=(Sanja - Tue, 01 Sep 2009, 15:18)=-=- testing Monty's patch Worked 3 hours and estimate 0 hours remain (original estimate increased by 1 hour). -=-=(Guest - Fri, 28 Aug 2009, 10:55)=-=- Low Level Design modified. --- /tmp/wklog.52.old.16945 2009-08-28 10:55:40.000000000 +0300 +++ /tmp/wklog.52.new.16945 2009-08-28 10:55:40.000000000 +0300 @@ -1 +1,22 @@ -Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. +Here is instructions of how to repeat this if needed: + +- Copy the following files from the latest MySQL 5.1 to MySQL 5.0: + +include/keycache.h +include/my_bit.h +mysys/mf_keycache.c +mysys/mf_keycaches.c +mysys/my_bit.c + +Apply the following change in mysys/mf_keycaches.c +81c81 +< static uchar *safe_hash_entry_get(SAFE_HASH_ENTRY *entry, size_t *length, +--- +> static uchar *safe_hash_entry_get(SAFE_HASH_ENTRY *entry, uint *length, + +recompile everything. + +Simple test to verify that this is working: +cd mysql-test +mysql-test-run --valgrind key_cache +(the --valgrind argument can be left out if you don't have valgrind installed) -=-=(Guest - Thu, 27 Aug 2009, 21:52)=-=- - Planning code (Email dicussions & discussions with Sanja): 2 hours. - Backport of code (2 hours). Notice that mysys/mf_keycaches.c::safe_hash_entry_get() is different from the code in MySQL 5.1 - Testing code with mysql-test-run (especially running the key_cache.test with valgrind) - Tested the server with sql-bench Worked 6 hours and estimate 2 hours remain (original estimate unchanged). -=-=(Sanja - Thu, 27 Aug 2009, 09:56)=-=- High-Level Specification modified. --- /tmp/wklog.52.old.20094 2009-08-27 09:56:05.000000000 +0300 +++ /tmp/wklog.52.new.20094 2009-08-27 09:56:05.000000000 +0300 @@ -1 +1 @@ - +Backport keycache to 5.0 -=-=(Sanja - Thu, 27 Aug 2009, 09:55)=-=- Low Level Design modified. --- /tmp/wklog.52.old.20035 2009-08-27 09:55:25.000000000 +0300 +++ /tmp/wklog.52.new.20035 2009-08-27 09:55:25.000000000 +0300 @@ -1 +1 @@ - +Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. DESCRIPTION: Backport keycache to bring bugfixes made in 5.1. HIGH-LEVEL SPECIFICATION: Backport keycache to 5.0 LOW-LEVEL DESIGN: Here is instructions of how to repeat this if needed: - Copy the following files from the latest MySQL 5.1 to MySQL 5.0: include/keycache.h include/my_bit.h mysys/mf_keycache.c mysys/mf_keycaches.c mysys/my_bit.c Apply the following change in mysys/mf_keycaches.c 81c81 < static uchar *safe_hash_entry_get(SAFE_HASH_ENTRY *entry, size_t *length, --- > static uchar *safe_hash_entry_get(SAFE_HASH_ENTRY *entry, uint *length, recompile everything. Simple test to verify that this is working: cd mysql-test mysql-test-run --valgrind key_cache (the --valgrind argument can be left out if you don't have valgrind installed) ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] [Branch ~maria-captains/maria/5.1] Rev 2723: Trivial fixes for windows compile failures
by noreply＠launchpad.net 01 Sep '09

01 Sep '09

------------------------------------------------------------ revno: 2723 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-fixwindows timestamp: Tue 2009-09-01 15:59:54 +0400 message: Trivial fixes for windows compile failures - Add strmov_overlapp.c where it's needed - Add libevent where it is required - Fix typo in unittest/mysys/waiting_threads-t.c - fix xtradb C language source so that they have variable declarations before any operators, as it is (or was?) required in C. modified: libmysql/CMakeLists.txt sql/CMakeLists.txt storage/xtradb/btr/btr0cur.c storage/xtradb/buf/buf0buddy.c storage/xtradb/buf/buf0flu.c storage/xtradb/fil/fil0fil.c storage/xtradb/handler/i_s.cc storage/xtradb/srv/srv0srv.c strings/CMakeLists.txt unittest/mysys/waiting_threads-t.c -- lp:maria https://code.launchpad.net/~maria-captains/maria/5.1 Your team Maria developers is subscribed to branch lp:maria. To unsubscribe from this branch go to https://code.launchpad.net/~maria-captains/maria/5.1/+edit-subscription.

1 0

[Maria-developers] Rev 2723: Trivial fixes for windows compile failures in file:///home/psergey/dev/maria-5.1-fixwindows/
by Sergey Petrunya 01 Sep '09

01 Sep '09

At file:///home/psergey/dev/maria-5.1-fixwindows/ ------------------------------------------------------------ revno: 2723 revision-id: psergey(a)askmonty.org-20090901115954-nge2brzh3akkmtlz parent: knielsen(a)knielsen-hq.org-20090831112339-td6t7ljy4w2oamvt committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-fixwindows timestamp: Tue 2009-09-01 15:59:54 +0400 message: Trivial fixes for windows compile failures - Add strmov_overlapp.c where it's needed - Add libevent where it is required - Fix typo in unittest/mysys/waiting_threads-t.c - fix xtradb C language source so that they have variable declarations before any operators, as it is (or was?) required in C. === modified file 'libmysql/CMakeLists.txt' --- a/libmysql/CMakeLists.txt 2008-04-28 16:24:05 +0000 +++ b/libmysql/CMakeLists.txt 2009-09-01 11:59:54 +0000 @@ -94,7 +94,7 @@ ../mysys/safemalloc.c ../mysys/sha1.c ../strings/str2int.c ../strings/str_alloc.c ../strings/strcend.c ../strings/strcont.c ../strings/strend.c ../strings/strfill.c ../mysys/string.c ../strings/strinstr.c ../strings/strmake.c - ../strings/strmov.c ../strings/strnlen.c ../strings/strnmov.c ../strings/strtod.c + ../strings/strmov.c ../strings/strmov_overlapp.c ../strings/strnlen.c ../strings/strnmov.c ../strings/strtod.c ../strings/strtoll.c ../strings/strtoull.c ../strings/strxmov.c ../strings/strxnmov.c ../mysys/thr_mutex.c ../mysys/typelib.c ../vio/vio.c ../vio/viosocket.c ../vio/viossl.c ../vio/viosslfactories.c ../strings/xml.c ../mysys/mf_qsort.c === modified file 'sql/CMakeLists.txt' --- a/sql/CMakeLists.txt 2008-11-21 14:21:50 +0000 +++ b/sql/CMakeLists.txt 2009-09-01 11:59:54 +0000 @@ -25,6 +25,7 @@ ${CMAKE_SOURCE_DIR}/sql ${CMAKE_SOURCE_DIR}/regex ${CMAKE_SOURCE_DIR}/zlib + ${CMAKE_SOURCE_DIR}/extra/libevent ) SET_SOURCE_FILES_PROPERTIES(${CMAKE_SOURCE_DIR}/sql/sql_yacc.h @@ -85,7 +86,7 @@ TARGET_LINK_LIBRARIES(mysqld heap myisam myisammrg mysys yassl zlib debug dbug yassl - taocrypt strings vio regex wsock32 ws2_32) + taocrypt strings vio regex libevent wsock32 ws2_32) SET_TARGET_PROPERTIES(mysqld PROPERTIES OUTPUT_NAME mysqld${MYSQLD_EXE_SUFFIX}) @@ -139,7 +140,7 @@ # Gen_lex_hash ADD_EXECUTABLE(gen_lex_hash gen_lex_hash.cc) -TARGET_LINK_LIBRARIES(gen_lex_hash debug dbug mysqlclient wsock32) +TARGET_LINK_LIBRARIES(gen_lex_hash debug dbug mysqlclient strings wsock32) GET_TARGET_PROPERTY(GEN_LEX_HASH_EXE gen_lex_hash LOCATION) ADD_CUSTOM_COMMAND( OUTPUT ${PROJECT_SOURCE_DIR}/sql/lex_hash.h === modified file 'storage/xtradb/btr/btr0cur.c' --- a/storage/xtradb/btr/btr0cur.c 2009-06-25 01:43:25 +0000 +++ b/storage/xtradb/btr/btr0cur.c 2009-09-01 11:59:54 +0000 @@ -3267,6 +3267,7 @@ } while (rec != supremum) { + rec_t* next_rec; /* count recs */ if (stats_method == SRV_STATS_METHOD_IGNORE_NULLS) { n_recs++; @@ -3281,7 +3282,7 @@ } } - rec_t* next_rec = page_rec_get_next(rec); + next_rec = page_rec_get_next(rec); if (next_rec == supremum) { break; } === modified file 'storage/xtradb/buf/buf0buddy.c' --- a/storage/xtradb/buf/buf0buddy.c 2009-06-25 01:43:25 +0000 +++ b/storage/xtradb/buf/buf0buddy.c 2009-09-01 11:59:54 +0000 @@ -461,11 +461,11 @@ actually is a properly initialized buf_page_t object. */ if (size >= PAGE_ZIP_MIN_SIZE) { + mutex_t* mutex; if (!have_page_hash_mutex) mutex_exit(&zip_free_mutex); /* This is a compressed page. */ - mutex_t* mutex; if (!have_page_hash_mutex) { mutex_enter(&LRU_list_mutex); === modified file 'storage/xtradb/buf/buf0flu.c' --- a/storage/xtradb/buf/buf0flu.c 2009-06-25 01:43:25 +0000 +++ b/storage/xtradb/buf/buf0flu.c 2009-09-01 11:59:54 +0000 @@ -1106,11 +1106,13 @@ page_count += buf_flush_try_neighbors( space, offset, flush_type); } else { + mutex_t* block_mutex; + buf_page_t* bpage_tmp; /* Try to flush the page only */ //buf_pool_mutex_enter(); rw_lock_s_lock(&page_hash_latch); - mutex_t* block_mutex = buf_page_get_mutex(bpage); + block_mutex = buf_page_get_mutex(bpage); retry_lock_2: mutex_enter(block_mutex); if (block_mutex != buf_page_get_mutex(bpage)) { @@ -1119,7 +1121,7 @@ goto retry_lock_2; } - buf_page_t* bpage_tmp = buf_page_hash_get(space, offset); + bpage_tmp = buf_page_hash_get(space, offset); if (bpage_tmp) { buf_flush_page(bpage_tmp, flush_type); page_count++; @@ -1238,13 +1240,14 @@ + BUF_FLUSH_EXTRA_MARGIN) && (distance < BUF_LRU_FREE_SEARCH_LEN)) { - if (!bpage->in_LRU_list) { + mutex_t* block_mutex; + if (!bpage->in_LRU_list) { /* reatart. but it is very optimistic */ bpage = UT_LIST_GET_LAST(buf_pool->LRU); continue; } - mutex_t* block_mutex = buf_page_get_mutex(bpage); + block_mutex = buf_page_get_mutex(bpage); retry_lock: mutex_enter(block_mutex); === modified file 'storage/xtradb/fil/fil0fil.c' --- a/storage/xtradb/fil/fil0fil.c 2009-06-25 01:43:25 +0000 +++ b/storage/xtradb/fil/fil0fil.c 2009-09-01 11:59:54 +0000 @@ -3039,6 +3039,13 @@ ulint i; int len; ib_uint64_t current_lsn; + ulint size_low, size_high, size; + ib_int64_t size_bytes; + dict_table_t* table; + dict_index_t* index; + fil_system_t* system; + fil_node_t* node = NULL; + fil_space_t* space; current_lsn = log_get_lsn(); @@ -3060,15 +3067,11 @@ success = os_file_write(filepath, file, page, 0, 0, UNIV_PAGE_SIZE); /* get file size */ - ulint size_low, size_high, size; - ib_int64_t size_bytes; os_file_get_size(file, &size_low, &size_high); size_bytes = (((ib_int64_t)size_high) << 32) + (ib_int64_t)size_low; /* get cruster index information */ - dict_table_t* table; - dict_index_t* index; table = dict_table_get_low(name); index = dict_table_get_first_index(table); ut_a(index->page==3); @@ -3076,9 +3079,9 @@ /* read metadata from .exp file */ n_index = 0; - bzero(old_id, sizeof(old_id)); - bzero(new_id, sizeof(new_id)); - bzero(root_page, sizeof(root_page)); + memset(old_id, 0, sizeof(old_id)); + memset(new_id, 0, sizeof(new_id)); + memset(root_page, 0, sizeof(root_page)); info_file_path = fil_make_ibd_name(name, FALSE); len = strlen(info_file_path); @@ -3128,9 +3131,9 @@ mem_heap_t* heap = NULL; ulint offsets_[REC_OFFS_NORMAL_SIZE]; ulint* offsets = offsets_; + ib_int64_t offset; size = (ulint) (size_bytes / UNIV_PAGE_SIZE); /* over write space id of all pages */ - ib_int64_t offset; rec_offs_init(offsets_); @@ -3284,10 +3287,8 @@ } mem_free(info_file_path); - fil_system_t* system = fil_system; + system = fil_system; mutex_enter(&(system->mutex)); - fil_node_t* node = NULL; - fil_space_t* space; space = fil_space_get_by_id(id); if (space) node = UT_LIST_GET_FIRST(space->chain); === modified file 'storage/xtradb/handler/i_s.cc' --- a/storage/xtradb/handler/i_s.cc 2009-06-25 01:43:25 +0000 +++ b/storage/xtradb/handler/i_s.cc 2009-09-01 11:59:54 +0000 @@ -763,7 +763,7 @@ dict_index_t* index; dulint index_id; - char *p; + const char *p; char db_name_raw[NAME_LEN*5+1], db_name[NAME_LEN+1]; char table_name_raw[NAME_LEN*5+1], table_name[NAME_LEN+1]; @@ -2861,7 +2861,7 @@ } else { rec_per_key = n_rows; } - snprintf(buff, 256, (i == index->n_uniq)?"%llu":"%llu, ", + ut_snprintf(buff, 256, (i == index->n_uniq)?"%llu":"%llu, ", rec_per_key); strncat(row_per_keys, buff, 256 - strlen(row_per_keys)); } === modified file 'storage/xtradb/srv/srv0srv.c' --- a/storage/xtradb/srv/srv0srv.c 2009-07-06 05:47:15 +0000 +++ b/storage/xtradb/srv/srv0srv.c 2009-09-01 11:59:54 +0000 @@ -2693,11 +2693,11 @@ /* defence line (max_checkpoint_age * 1/2) */ ib_uint64_t lsn = log_sys->lsn; - mutex_exit(&(log_sys->mutex)); - ib_uint64_t level, bpl; buf_page_t* bpage; + mutex_exit(&(log_sys->mutex)); + mutex_enter(&flush_list_mutex); level = 0; === modified file 'strings/CMakeLists.txt' --- a/strings/CMakeLists.txt 2008-10-10 15:28:41 +0000 +++ b/strings/CMakeLists.txt 2009-09-01 11:59:54 +0000 @@ -23,7 +23,7 @@ ctype-latin1.c ctype-mb.c ctype-simple.c ctype-sjis.c ctype-tis620.c ctype-uca.c ctype-ucs2.c ctype-ujis.c ctype-utf8.c ctype-win1250ch.c ctype.c decimal.c int2str.c is_prefix.c llstr.c longlong2str.c my_strtoll10.c my_vsnprintf.c r_strinstr.c - str2int.c str_alloc.c strcend.c strend.c strfill.c strmake.c strmov.c strnmov.c + str2int.c str_alloc.c strcend.c strend.c strfill.c strmake.c strmov.c strmov_overlapp.c strnmov.c strtod.c strtol.c strtoll.c strtoul.c strtoull.c strxmov.c strxnmov.c xml.c my_strchr.c strcont.c strinstr.c strnlen.c strappend.c) === modified file 'unittest/mysys/waiting_threads-t.c' --- a/unittest/mysys/waiting_threads-t.c 2009-03-20 11:41:30 +0000 +++ b/unittest/mysys/waiting_threads-t.c 2009-09-01 11:59:54 +0000 @@ -263,6 +263,7 @@ kill_strategy=X; \ do_one_test(); #else +#define test_kill_strategy(X) \ diag("kill strategy: " #X); \ DBUG_PRINT("info", ("kill strategy: " #X)); \ kill_strategy=X; \

1 0

[Maria-developers] Rev 2720: Fix for in file:///Users/hakan/work/monty_program/maria/
by Hakan Kuecuekyilmaz 31 Aug '09

31 Aug '09

At file:///Users/hakan/work/monty_program/maria/ ------------------------------------------------------------ revno: 2720 revision-id: hakan(a)askmonty.org-20090831214839-gsh1r8mnkrdyzc38 parent: knielsen(a)knielsen-hq.org-20090805072137-wg97dcem1cxnzt3p committer: Hakan Kuecuekyilmaz <hakan(a)askmonty.org> branch nick: maria timestamp: Mon 2009-08-31 23:48:39 +0200 message: Fix for mysqlslap: setting --engine does not get replicated http://bugs.mysql.com/bug.php?id=46967 Problem was that a "set storage_engine = <storage_engine>" is not replicated. Fixed by adding an explicit Engine = <storage_engine> to the build_table_string() function. Open question: Should MySQL's replication actually replicate the set storage_engine command or not? === modified file 'client/mysqlslap.c' --- a/client/mysqlslap.c 2009-04-25 10:05:32 +0000 +++ b/client/mysqlslap.c 2009-08-31 21:48:39 +0000 @@ -880,6 +880,12 @@ } dynstr_append(&table_string, ")"); + + if (default_engine) { + dynstr_append(&table_string, " Engine = "); + dynstr_append(&table_string, default_engine); + } + ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL|MY_FAE|MY_WME)); ptr->string = (char *)my_malloc(table_string.length+1,

1 0

[Maria-developers] Rev 2720: Fix for in file:///Users/hakan/work/monty_program/maria/
by Hakan Kuecuekyilmaz 31 Aug '09

31 Aug '09

At file:///Users/hakan/work/monty_program/maria/ ------------------------------------------------------------ revno: 2720 revision-id: hakan(a)askmonty.org-20090831212214-wg9fwhgnhx1ly6w2 parent: knielsen(a)knielsen-hq.org-20090805072137-wg97dcem1cxnzt3p committer: Hakan Kuecuekyilmaz <hakan(a)askmonty.org> branch nick: maria timestamp: Mon 2009-08-31 23:22:14 +0200 message: Fix for mysqlslap: setting --engine does not get replicated http://bugs.mysql.com/bug.php?id=46967 Problem was that a "set storage_engine = <storage_engine>" is not replicated. Fixed by adding an explicit Engine = <storage_engine> to the build_table_string() function. Open question: Should MySQL's replication actually replicate the set storage_engine command or not? === modified file 'client/mysqlslap.c' --- a/client/mysqlslap.c 2009-04-25 10:05:32 +0000 +++ b/client/mysqlslap.c 2009-08-31 21:22:14 +0000 @@ -880,6 +880,12 @@ } dynstr_append(&table_string, ")"); + + if (default_engine) { + dynstr_append(&table_string, " Engine = "); + dynstr_append(&table_string, default_engine); + } + ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL|MY_FAE|MY_WME)); ptr->string = (char *)my_malloc(table_string.length+1,

1 0

[Maria-developers] [Branch ~maria-captains/maria/5.1] Rev 2722: Merge Paul's maria-pbxt-rc2 branch into MariaDB.
by noreply＠launchpad.net 31 Aug '09

31 Aug '09

Merge authors: Kristian Nielsen (knielsen) Paul McCullagh (paul-mccullagh) Related merge proposals: https://code.launchpad.net/~paul-mccullagh/maria/maria-pbxt-rc2/+merge/10303 proposed by: Paul McCullagh (paul-mccullagh) ------------------------------------------------------------ revno: 2722 [merge] committer: knielsen(a)knielsen-hq.org branch nick: work timestamp: Mon 2009-08-31 13:23:39 +0200 message: Merge Paul's maria-pbxt-rc2 branch into MariaDB. Includes disabling PBMS (per Paul's recommendation) and test file fixes. Signed-off-by: Kristian Nielsen <knielsen(a)knielsen-hq.org> removed: storage/pbxt/src/streaming_xt.cc storage/pbxt/src/streaming_xt.h added: mysql-test/suite/pbxt/t/lowercase_table_grant-master.opt mysql-test/suite/pbxt/t/lowercase_table_qcache-master.opt mysql-test/suite/pbxt/t/lowercase_view-master.opt mysql-test/suite/pbxt/t/udf-master.opt storage/pbxt/src/pbms_enabled.cc storage/pbxt/src/pbms_enabled.h modified: mysql-test/suite/pbxt/r/alter_table.result mysql-test/suite/pbxt/r/analyze.result mysql-test/suite/pbxt/r/auto_increment.result mysql-test/suite/pbxt/r/delete.result mysql-test/suite/pbxt/r/distinct.result mysql-test/suite/pbxt/r/func_group.result mysql-test/suite/pbxt/r/func_math.result mysql-test/suite/pbxt/r/func_str.result mysql-test/suite/pbxt/r/grant.result mysql-test/suite/pbxt/r/group_min_max.result mysql-test/suite/pbxt/r/join.result mysql-test/suite/pbxt/r/join_nested.result mysql-test/suite/pbxt/r/key.result mysql-test/suite/pbxt/r/key_cache.result mysql-test/suite/pbxt/r/key_diff.result mysql-test/suite/pbxt/r/lowercase_view.result mysql-test/suite/pbxt/r/mysqlshow.result mysql-test/suite/pbxt/r/null.result mysql-test/suite/pbxt/r/null_key.result mysql-test/suite/pbxt/r/partition_pruning.result mysql-test/suite/pbxt/r/pbxt_bugs.result mysql-test/suite/pbxt/r/pbxt_ref_int.result mysql-test/suite/pbxt/r/preload.result mysql-test/suite/pbxt/r/ps_1general.result mysql-test/suite/pbxt/r/range.result mysql-test/suite/pbxt/r/schema.result mysql-test/suite/pbxt/r/select.result mysql-test/suite/pbxt/r/select_safe.result mysql-test/suite/pbxt/r/subselect.result mysql-test/suite/pbxt/r/type_enum.result mysql-test/suite/pbxt/r/type_ranges.result mysql-test/suite/pbxt/r/type_timestamp.result mysql-test/suite/pbxt/r/union.result mysql-test/suite/pbxt/r/view_grant.result mysql-test/suite/pbxt/t/auto_increment.test mysql-test/suite/pbxt/t/delete.test mysql-test/suite/pbxt/t/join_nested.test mysql-test/suite/pbxt/t/null.test mysql-test/suite/pbxt/t/pbxt_bugs.test mysql-test/suite/pbxt/t/rename.test mysql-test/suite/pbxt/t/schema.test mysql-test/suite/pbxt/t/type_enum.test mysql-test/suite/pbxt/t/union.test storage/pbxt/ChangeLog storage/pbxt/src/Makefile.am storage/pbxt/src/cache_xt.cc storage/pbxt/src/cache_xt.h storage/pbxt/src/ccutils_xt.cc storage/pbxt/src/database_xt.cc storage/pbxt/src/datadic_xt.cc storage/pbxt/src/datadic_xt.h storage/pbxt/src/datalog_xt.cc storage/pbxt/src/datalog_xt.h storage/pbxt/src/discover_xt.cc storage/pbxt/src/filesys_xt.cc storage/pbxt/src/filesys_xt.h storage/pbxt/src/ha_pbxt.cc storage/pbxt/src/ha_pbxt.h storage/pbxt/src/ha_xtsys.cc storage/pbxt/src/ha_xtsys.h storage/pbxt/src/hashtab_xt.cc storage/pbxt/src/heap_xt.cc storage/pbxt/src/heap_xt.h storage/pbxt/src/index_xt.cc storage/pbxt/src/index_xt.h storage/pbxt/src/lock_xt.cc storage/pbxt/src/lock_xt.h storage/pbxt/src/locklist_xt.cc storage/pbxt/src/locklist_xt.h storage/pbxt/src/memory_xt.cc storage/pbxt/src/memory_xt.h storage/pbxt/src/myxt_xt.cc storage/pbxt/src/myxt_xt.h storage/pbxt/src/pbms.h storage/pbxt/src/pthread_xt.cc storage/pbxt/src/restart_xt.cc storage/pbxt/src/restart_xt.h storage/pbxt/src/sortedlist_xt.cc storage/pbxt/src/strutil_xt.cc storage/pbxt/src/systab_xt.cc storage/pbxt/src/systab_xt.h storage/pbxt/src/tabcache_xt.cc storage/pbxt/src/tabcache_xt.h storage/pbxt/src/table_xt.cc storage/pbxt/src/table_xt.h storage/pbxt/src/thread_xt.cc storage/pbxt/src/thread_xt.h storage/pbxt/src/trace_xt.cc storage/pbxt/src/trace_xt.h storage/pbxt/src/util_xt.cc storage/pbxt/src/xaction_xt.cc storage/pbxt/src/xaction_xt.h storage/pbxt/src/xactlog_xt.cc storage/pbxt/src/xactlog_xt.h storage/pbxt/src/xt_config.h storage/pbxt/src/xt_defs.h The size of the diff (15366 lines) is larger than your specified limit of 1000 lines -- lp:maria https://code.launchpad.net/~maria-captains/maria/5.1 Your team Maria developers is subscribed to branch lp:maria. To unsubscribe from this branch go to https://code.launchpad.net/~maria-captains/maria/5.1/+edit-subscription.

1 0

[Maria-developers] Rev 2749: MWL#17: Table-elimination in file:///home/psergey/dev/maria-5.1-table-elim-r11/
by Sergey Petrunya 31 Aug '09

31 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r11/ ------------------------------------------------------------ revno: 2749 revision-id: psergey(a)askmonty.org-20090831200209-7gbnkr7omjbgld1x parent: psergey(a)askmonty.org-20090826210140-y2pba3ucdwnvr956 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Tue 2009-09-01 00:02:09 +0400 message: MWL#17: Table-elimination - Addressing review feedback, generation 4. Diff too large for email (2658 lines, the limit is 1000).

1 0

[Maria-developers] MariaDB versioning
by Peter Laursen 31 Aug '09

31 Aug '09

I have a concern that I will ask you all to consider while there is time. I think I understand that after MariaDB 5.1 a 5.2 release is planned. I think it should not be versioned like that. Problem is that there is a MySQL 5.2 tree too (it is basically an early 6.0) and it is still available for download from FTP-mirrors like ftp://mirrors.dotsrc.org/mysql/Downloads/ . . For generic and GUI clients (ie. not designed for specific server versions and where SQL is generated and not written by user) it is important that the version string can be used to identify what SQL statements are supported by server. The first thing such clients (like our own SQLyog) does after connection is to SELECT VERSION() .. and only after that we know if there is support for EVENTS, if IF EXISTS is supported for DROP TRIGGER sysntax and so on. If there are two different 5.2 trees (MariaDB based on MySQL 5.1 and MySQL 5.2) it gets more difficult to handle. . I asked Monty about this at one of his sessions at the latest UC and he told like 'we are good citizens - except for the comment field MariaDB will be versioned like the MySQL code it is based upon'. What I also think is the best solution. The *version* should not be the same for different builds with even the slightest difference in what SQL server understands. On the opposite version should be the same as long as SQL- interface (even with possible Engine-specific extensions) is the same no matter how much they differ internally. Maybe in the long run diversification is inevitable. But in that case it should be planned well! Peter Webyog -- This message was sent from Launchpad by the user Peter Laursen (https://launchpad.net/~peter-laursen) using the "Contact this team" link on the Maria Discuss team page to each member directly. For more information see https://help.launchpad.net/YourAccount/ContactingPeople

3 3

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2722)
by knielsen＠knielsen-hq.org 31 Aug '09

31 Aug '09

#At lp:maria 2722 knielsen(a)knielsen-hq.org 2009-08-31 [merge] Merge Paul's maria-pbxt-rc2 branch into MariaDB. Includes disabling PBMS (per Paul's recommendation) and test file fixes. Signed-off-by: Kristian Nielsen <knielsen(a)knielsen-hq.org> removed: storage/pbxt/src/streaming_xt.cc storage/pbxt/src/streaming_xt.h added: mysql-test/suite/pbxt/t/lowercase_table_grant-master.opt mysql-test/suite/pbxt/t/lowercase_table_qcache-master.opt mysql-test/suite/pbxt/t/lowercase_view-master.opt mysql-test/suite/pbxt/t/udf-master.opt storage/pbxt/src/pbms_enabled.cc storage/pbxt/src/pbms_enabled.h modified: mysql-test/suite/pbxt/r/alter_table.result mysql-test/suite/pbxt/r/analyze.result mysql-test/suite/pbxt/r/auto_increment.result mysql-test/suite/pbxt/r/delete.result mysql-test/suite/pbxt/r/distinct.result mysql-test/suite/pbxt/r/func_group.result mysql-test/suite/pbxt/r/func_math.result mysql-test/suite/pbxt/r/func_str.result mysql-test/suite/pbxt/r/grant.result mysql-test/suite/pbxt/r/group_min_max.result mysql-test/suite/pbxt/r/join.result mysql-test/suite/pbxt/r/join_nested.result mysql-test/suite/pbxt/r/key.result mysql-test/suite/pbxt/r/key_cache.result mysql-test/suite/pbxt/r/key_diff.result mysql-test/suite/pbxt/r/lowercase_view.result mysql-test/suite/pbxt/r/mysqlshow.result mysql-test/suite/pbxt/r/null.result mysql-test/suite/pbxt/r/null_key.result mysql-test/suite/pbxt/r/partition_pruning.result mysql-test/suite/pbxt/r/pbxt_bugs.result mysql-test/suite/pbxt/r/pbxt_ref_int.result mysql-test/suite/pbxt/r/preload.result mysql-test/suite/pbxt/r/ps_1general.result mysql-test/suite/pbxt/r/range.result mysql-test/suite/pbxt/r/schema.result mysql-test/suite/pbxt/r/select.result mysql-test/suite/pbxt/r/select_safe.result mysql-test/suite/pbxt/r/subselect.result mysql-test/suite/pbxt/r/type_enum.result mysql-test/suite/pbxt/r/type_ranges.result mysql-test/suite/pbxt/r/type_timestamp.result mysql-test/suite/pbxt/r/union.result mysql-test/suite/pbxt/r/view_grant.result mysql-test/suite/pbxt/t/auto_increment.test mysql-test/suite/pbxt/t/delete.test mysql-test/suite/pbxt/t/join_nested.test mysql-test/suite/pbxt/t/null.test mysql-test/suite/pbxt/t/pbxt_bugs.test mysql-test/suite/pbxt/t/rename.test mysql-test/suite/pbxt/t/schema.test mysql-test/suite/pbxt/t/type_enum.test mysql-test/suite/pbxt/t/union.test storage/pbxt/ChangeLog storage/pbxt/src/Makefile.am storage/pbxt/src/cache_xt.cc storage/pbxt/src/cache_xt.h storage/pbxt/src/ccutils_xt.cc storage/pbxt/src/database_xt.cc storage/pbxt/src/datadic_xt.cc storage/pbxt/src/datadic_xt.h storage/pbxt/src/datalog_xt.cc storage/pbxt/src/datalog_xt.h storage/pbxt/src/discover_xt.cc storage/pbxt/src/filesys_xt.cc storage/pbxt/src/filesys_xt.h storage/pbxt/src/ha_pbxt.cc storage/pbxt/src/ha_pbxt.h storage/pbxt/src/ha_xtsys.cc storage/pbxt/src/ha_xtsys.h storage/pbxt/src/hashtab_xt.cc storage/pbxt/src/heap_xt.cc storage/pbxt/src/heap_xt.h storage/pbxt/src/index_xt.cc storage/pbxt/src/index_xt.h storage/pbxt/src/lock_xt.cc storage/pbxt/src/lock_xt.h storage/pbxt/src/locklist_xt.cc storage/pbxt/src/locklist_xt.h storage/pbxt/src/memory_xt.cc storage/pbxt/src/memory_xt.h storage/pbxt/src/myxt_xt.cc storage/pbxt/src/myxt_xt.h storage/pbxt/src/pbms.h storage/pbxt/src/pthread_xt.cc storage/pbxt/src/restart_xt.cc storage/pbxt/src/restart_xt.h storage/pbxt/src/sortedlist_xt.cc storage/pbxt/src/strutil_xt.cc storage/pbxt/src/systab_xt.cc storage/pbxt/src/systab_xt.h storage/pbxt/src/tabcache_xt.cc storage/pbxt/src/tabcache_xt.h storage/pbxt/src/table_xt.cc storage/pbxt/src/table_xt.h storage/pbxt/src/thread_xt.cc storage/pbxt/src/thread_xt.h storage/pbxt/src/trace_xt.cc storage/pbxt/src/trace_xt.h storage/pbxt/src/util_xt.cc storage/pbxt/src/xaction_xt.cc storage/pbxt/src/xaction_xt.h storage/pbxt/src/xactlog_xt.cc storage/pbxt/src/xactlog_xt.h storage/pbxt/src/xt_config.h storage/pbxt/src/xt_defs.h === modified file 'mysql-test/suite/pbxt/r/alter_table.result' --- a/mysql-test/suite/pbxt/r/alter_table.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/alter_table.result 2009-08-17 11:12:36 +0000 @@ -126,23 +126,23 @@ key (n4, n1, n2, n3) ); alter table t1 disable keys; show keys from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -t1 0 n1 1 n1 NULL 0 NULL NULL BTREE -t1 1 n1_2 1 n1 NULL NULL NULL NULL BTREE -t1 1 n1_2 2 n2 NULL NULL NULL NULL YES BTREE -t1 1 n1_2 3 n3 NULL NULL NULL NULL YES BTREE -t1 1 n1_2 4 n4 NULL NULL NULL NULL YES BTREE -t1 1 n2 1 n2 NULL NULL NULL NULL YES BTREE -t1 1 n2 2 n3 NULL NULL NULL NULL YES BTREE -t1 1 n2 3 n4 NULL NULL NULL NULL YES BTREE -t1 1 n2 4 n1 NULL NULL NULL NULL BTREE -t1 1 n3 1 n3 NULL NULL NULL NULL YES BTREE -t1 1 n3 2 n4 NULL NULL NULL NULL YES BTREE -t1 1 n3 3 n1 NULL NULL NULL NULL BTREE -t1 1 n3 4 n2 NULL NULL NULL NULL YES BTREE -t1 1 n4 1 n4 NULL NULL NULL NULL YES BTREE -t1 1 n4 2 n1 NULL NULL NULL NULL BTREE -t1 1 n4 3 n2 NULL NULL NULL NULL YES BTREE -t1 1 n4 4 n3 NULL NULL NULL NULL YES BTREE +t1 0 n1 1 n1 A 0 NULL NULL BTREE +t1 1 n1_2 1 n1 A 0 NULL NULL BTREE +t1 1 n1_2 2 n2 A 0 NULL NULL YES BTREE +t1 1 n1_2 3 n3 A 0 NULL NULL YES BTREE +t1 1 n1_2 4 n4 A 0 NULL NULL YES BTREE +t1 1 n2 1 n2 A 0 NULL NULL YES BTREE +t1 1 n2 2 n3 A 0 NULL NULL YES BTREE +t1 1 n2 3 n4 A 0 NULL NULL YES BTREE +t1 1 n2 4 n1 A 0 NULL NULL BTREE +t1 1 n3 1 n3 A 0 NULL NULL YES BTREE +t1 1 n3 2 n4 A 0 NULL NULL YES BTREE +t1 1 n3 3 n1 A 0 NULL NULL BTREE +t1 1 n3 4 n2 A 0 NULL NULL YES BTREE +t1 1 n4 1 n4 A 0 NULL NULL YES BTREE +t1 1 n4 2 n1 A 0 NULL NULL BTREE +t1 1 n4 3 n2 A 0 NULL NULL YES BTREE +t1 1 n4 4 n3 A 0 NULL NULL YES BTREE insert into t1 values(10,RAND()*1000,RAND()*1000,RAND()); insert into t1 values(9,RAND()*1000,RAND()*1000,RAND()); insert into t1 values(8,RAND()*1000,RAND()*1000,RAND()); @@ -156,23 +156,23 @@ insert into t1 values(1,RAND()*1000,RAND alter table t1 enable keys; show keys from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -t1 0 n1 1 n1 NULL 10 NULL NULL BTREE -t1 1 n1_2 1 n1 NULL NULL NULL NULL BTREE -t1 1 n1_2 2 n2 NULL NULL NULL NULL YES BTREE -t1 1 n1_2 3 n3 NULL NULL NULL NULL YES BTREE -t1 1 n1_2 4 n4 NULL NULL NULL NULL YES BTREE -t1 1 n2 1 n2 NULL NULL NULL NULL YES BTREE -t1 1 n2 2 n3 NULL NULL NULL NULL YES BTREE -t1 1 n2 3 n4 NULL NULL NULL NULL YES BTREE -t1 1 n2 4 n1 NULL NULL NULL NULL BTREE -t1 1 n3 1 n3 NULL NULL NULL NULL YES BTREE -t1 1 n3 2 n4 NULL NULL NULL NULL YES BTREE -t1 1 n3 3 n1 NULL NULL NULL NULL BTREE -t1 1 n3 4 n2 NULL NULL NULL NULL YES BTREE -t1 1 n4 1 n4 NULL NULL NULL NULL YES BTREE -t1 1 n4 2 n1 NULL NULL NULL NULL BTREE -t1 1 n4 3 n2 NULL NULL NULL NULL YES BTREE -t1 1 n4 4 n3 NULL NULL NULL NULL YES BTREE +t1 0 n1 1 n1 A 10 NULL NULL BTREE +t1 1 n1_2 1 n1 A 10 NULL NULL BTREE +t1 1 n1_2 2 n2 A 10 NULL NULL YES BTREE +t1 1 n1_2 3 n3 A 10 NULL NULL YES BTREE +t1 1 n1_2 4 n4 A 10 NULL NULL YES BTREE +t1 1 n2 1 n2 A 10 NULL NULL YES BTREE +t1 1 n2 2 n3 A 10 NULL NULL YES BTREE +t1 1 n2 3 n4 A 10 NULL NULL YES BTREE +t1 1 n2 4 n1 A 10 NULL NULL BTREE +t1 1 n3 1 n3 A 10 NULL NULL YES BTREE +t1 1 n3 2 n4 A 10 NULL NULL YES BTREE +t1 1 n3 3 n1 A 10 NULL NULL BTREE +t1 1 n3 4 n2 A 10 NULL NULL YES BTREE +t1 1 n4 1 n4 A 10 NULL NULL YES BTREE +t1 1 n4 2 n1 A 10 NULL NULL BTREE +t1 1 n4 3 n2 A 10 NULL NULL YES BTREE +t1 1 n4 4 n3 A 10 NULL NULL YES BTREE drop table t1; create table t1 (i int unsigned not null auto_increment primary key); alter table t1 rename t2; @@ -286,17 +286,17 @@ insert into t1 values(1,1), (2,1), (3, 1 alter table t1 add unique (a,b), add key (b); show keys from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -t1 0 a 1 a A NULL NULL NULL YES BTREE -t1 0 a 2 b A NULL NULL NULL YES BTREE -t1 1 b 1 b A NULL NULL NULL YES BTREE +t1 0 a 1 a A 300 NULL NULL YES BTREE +t1 0 a 2 b A 300 NULL NULL YES BTREE +t1 1 b 1 b A 300 NULL NULL YES BTREE analyze table t1; Table Op Msg_type Msg_text test.t1 analyze status OK show keys from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -t1 0 a 1 a A NULL NULL NULL YES BTREE -t1 0 a 2 b A NULL NULL NULL YES BTREE -t1 1 b 1 b A NULL NULL NULL YES BTREE +t1 0 a 1 a A 300 NULL NULL YES BTREE +t1 0 a 2 b A 300 NULL NULL YES BTREE +t1 1 b 1 b A 300 NULL NULL YES BTREE drop table t1; CREATE TABLE t1 (i int(10), index(i) ); ALTER TABLE t1 DISABLE KEYS; @@ -545,37 +545,37 @@ drop table if exists t1; create table t1 (a int, key(a)); show indexes from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -t1 1 a 1 a A NULL NULL NULL YES BTREE +t1 1 a 1 a A 0 NULL NULL YES BTREE "this used not to disable the index" alter table t1 modify a int, disable keys; show indexes from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -t1 1 a 1 a A NULL NULL NULL YES BTREE +t1 1 a 1 a A 0 NULL NULL YES BTREE alter table t1 enable keys; show indexes from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -t1 1 a 1 a NULL NULL NULL NULL YES BTREE +t1 1 a 1 a A 0 NULL NULL YES BTREE alter table t1 modify a bigint, disable keys; show indexes from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -t1 1 a 1 a A NULL NULL NULL YES BTREE +t1 1 a 1 a A 0 NULL NULL YES BTREE alter table t1 enable keys; show indexes from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -t1 1 a 1 a NULL NULL NULL NULL YES BTREE +t1 1 a 1 a A 0 NULL NULL YES BTREE alter table t1 add b char(10), disable keys; show indexes from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -t1 1 a 1 a A NULL NULL NULL YES BTREE +t1 1 a 1 a A 0 NULL NULL YES BTREE alter table t1 add c decimal(10,2), enable keys; show indexes from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -t1 1 a 1 a A NULL NULL NULL YES BTREE +t1 1 a 1 a A 0 NULL NULL YES BTREE "this however did" alter table t1 disable keys; show indexes from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -t1 1 a 1 a NULL NULL NULL NULL YES BTREE +t1 1 a 1 a A 0 NULL NULL YES BTREE desc t1; Field Type Null Key Default Extra a bigint(20) YES MUL NULL @@ -585,7 +585,7 @@ alter table t1 add d decimal(15,5); "The key should still be disabled" show indexes from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -t1 1 a 1 a A NULL NULL NULL YES BTREE +t1 1 a 1 a A 0 NULL NULL YES BTREE drop table t1; "Now will test with one unique index" create table t1(a int, b char(10), unique(a)); @@ -595,7 +595,7 @@ t1 0 a 1 a A 0 NULL NULL YES BTREE alter table t1 disable keys; show indexes from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -t1 0 a 1 a NULL 0 NULL NULL YES BTREE +t1 0 a 1 a A 0 NULL NULL YES BTREE alter table t1 enable keys; "If no copy on noop change, this won't touch the data file" "Unique index, no change" @@ -623,12 +623,12 @@ create table t1(a int, b char(10), uniqu show indexes from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment t1 0 a 1 a A 0 NULL NULL YES BTREE -t1 1 b 1 b A NULL NULL NULL YES BTREE +t1 1 b 1 b A 0 NULL NULL YES BTREE alter table t1 disable keys; show indexes from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -t1 0 a 1 a NULL 0 NULL NULL YES BTREE -t1 1 b 1 b NULL NULL NULL NULL YES BTREE +t1 0 a 1 a A 0 NULL NULL YES BTREE +t1 1 b 1 b A 0 NULL NULL YES BTREE alter table t1 enable keys; "If no copy on noop change, this won't touch the data file" "The non-unique index will be disabled" @@ -636,31 +636,31 @@ alter table t1 modify a int, disable key show indexes from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment t1 0 a 1 a A 0 NULL NULL YES BTREE -t1 1 b 1 b A NULL NULL NULL YES BTREE +t1 1 b 1 b A 0 NULL NULL YES BTREE alter table t1 enable keys; show indexes from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -t1 0 a 1 a NULL 0 NULL NULL YES BTREE -t1 1 b 1 b NULL NULL NULL NULL YES BTREE +t1 0 a 1 a A 0 NULL NULL YES BTREE +t1 1 b 1 b A 0 NULL NULL YES BTREE "Change the type implying data copy" "The non-unique index will be disabled" alter table t1 modify a bigint, disable keys; show indexes from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment t1 0 a 1 a A 0 NULL NULL YES BTREE -t1 1 b 1 b A NULL NULL NULL YES BTREE +t1 1 b 1 b A 0 NULL NULL YES BTREE "Change again the type, but leave the indexes as_is" alter table t1 modify a int; show indexes from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment t1 0 a 1 a A 0 NULL NULL YES BTREE -t1 1 b 1 b A NULL NULL NULL YES BTREE +t1 1 b 1 b A 0 NULL NULL YES BTREE "Try the same. When data is no copied on similar tables, this is noop" alter table t1 modify a int; show indexes from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment t1 0 a 1 a A 0 NULL NULL YES BTREE -t1 1 b 1 b A NULL NULL NULL YES BTREE +t1 1 b 1 b A 0 NULL NULL YES BTREE drop table t1; create database mysqltest; create table t1 (c1 int); @@ -697,11 +697,11 @@ DROP TABLE IF EXISTS bug24219_2; CREATE TABLE bug24219 (a INT, INDEX(a)); SHOW INDEX FROM bug24219; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -bug24219 1 a 1 a A NULL NULL NULL YES BTREE +bug24219 1 a 1 a A 0 NULL NULL YES BTREE ALTER TABLE bug24219 RENAME TO bug24219_2, DISABLE KEYS; SHOW INDEX FROM bug24219_2; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -bug24219_2 1 a 1 a A NULL NULL NULL YES BTREE +bug24219_2 1 a 1 a A 0 NULL NULL YES BTREE DROP TABLE bug24219_2; create table t1 (mycol int(10) not null); alter table t1 alter column mycol set default 0; @@ -882,7 +882,7 @@ int_field int(10) unsigned NO MUL NULL char_field char(10) YES NULL SHOW INDEXES FROM t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -t1 1 int_field 1 int_field A NULL NULL NULL BTREE +t1 1 int_field 1 int_field A 0 NULL NULL BTREE INSERT INTO t1 VALUES (1, "edno"), (1, "edno"), (2, "dve"), (3, "tri"), (5, "pet"); "Non-copy data change - new frm, but old data and index files" ALTER TABLE t1 === modified file 'mysql-test/suite/pbxt/r/analyze.result' --- a/mysql-test/suite/pbxt/r/analyze.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/analyze.result 2009-08-17 15:57:58 +0000 @@ -56,5 +56,5 @@ Table Op Msg_type Msg_text test.t1 analyze status OK show index from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment -t1 1 a 1 a A NULL NULL NULL YES BTREE +t1 1 a 1 a A 5 NULL NULL YES BTREE drop table t1; === modified file 'mysql-test/suite/pbxt/r/auto_increment.result' --- a/mysql-test/suite/pbxt/r/auto_increment.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/auto_increment.result 2009-08-17 15:57:58 +0000 @@ -229,7 +229,8 @@ a b 204 7 delete from t1 where a=0; update t1 set a=NULL where b=6; -ERROR 23000: Column 'a' cannot be null +Warnings: +Warning 1048 Column 'a' cannot be null update t1 set a=300 where b=7; SET SQL_MODE=''; insert into t1(a,b)values(NULL,8); @@ -244,7 +245,7 @@ a b 1 1 200 2 201 4 -203 6 +0 6 300 7 301 8 400 9 @@ -260,7 +261,6 @@ a b 1 1 200 2 201 4 -203 6 300 7 301 8 400 9 @@ -271,20 +271,20 @@ a b 405 14 delete from t1 where a=0; update t1 set a=NULL where b=13; -ERROR 23000: Column 'a' cannot be null +Warnings: +Warning 1048 Column 'a' cannot be null update t1 set a=500 where b=14; select * from t1 order by b; a b 1 1 200 2 201 4 -203 6 300 7 301 8 400 9 401 10 402 11 -404 13 +0 13 500 14 drop table t1; create table t1 (a bigint); === modified file 'mysql-test/suite/pbxt/r/delete.result' --- a/mysql-test/suite/pbxt/r/delete.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/delete.result 2009-08-17 15:57:58 +0000 @@ -125,18 +125,19 @@ a b 0 11 2 12 delete ignore t11.*, t12.* from t11,t12 where t11.a = t12.a and t11.b <> (select b from t2 where t11.a < t2.a); -Warnings: -Error 1242 Subquery returns more than 1 row -Error 1242 Subquery returns more than 1 row +ERROR 21000: Subquery returns more than 1 row select * from t11; a b 0 10 1 11 +2 12 select * from t12; a b 33 10 0 11 +2 12 insert into t11 values (2, 12); +ERROR 23000: Duplicate entry '2' for key 'PRIMARY' delete from t11 where t11.b <> (select b from t2 where t11.a < t2.a); ERROR 21000: Subquery returns more than 1 row select * from t11; @@ -145,13 +146,12 @@ a b 1 11 2 12 delete ignore from t11 where t11.b <> (select b from t2 where t11.a < t2.a); -Warnings: -Error 1242 Subquery returns more than 1 row -Error 1242 Subquery returns more than 1 row +ERROR 21000: Subquery returns more than 1 row select * from t11; a b 0 10 1 11 +2 12 drop table t11, t12, t2; create table t1 (a int, b int, unique key (a), key (b)); insert into t1 values (3, 3), (7, 7); === modified file 'mysql-test/suite/pbxt/r/distinct.result' --- a/mysql-test/suite/pbxt/r/distinct.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/distinct.result 2009-08-17 15:57:58 +0000 @@ -174,8 +174,8 @@ INSERT INTO t3 VALUES (1,'1'),(2,'2'),(1 explain SELECT distinct t3.a FROM t3,t2,t1 WHERE t3.a=t1.b AND t1.a=t2.a; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL PRIMARY NULL NULL NULL 4 Using temporary -1 SIMPLE t3 ref a a 5 test.t1.b 2 Using where; Using index -1 SIMPLE t2 index a a 4 NULL 5 Using where; Using index; Distinct; Using join buffer +1 SIMPLE t2 ref a a 4 test.t1.a 1 Using index +1 SIMPLE t3 ref a a 5 test.t1.b 1 Using where; Using index SELECT distinct t3.a FROM t3,t2,t1 WHERE t3.a=t1.b AND t1.a=t2.a; a 1 @@ -190,7 +190,7 @@ insert into t3 select * from t4; explain select distinct t1.a from t1,t3 where t1.a=t3.a; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 index PRIMARY PRIMARY 4 NULL 4 Using index; Using temporary -1 SIMPLE t3 ref a a 5 test.t1.a 11 Using where; Using index; Distinct +1 SIMPLE t3 ref a a 5 test.t1.a 1 Using where; Using index; Distinct select distinct t1.a from t1,t3 where t1.a=t3.a; a 1 @@ -212,7 +212,7 @@ id select_type table type possible_keys 1 SIMPLE t1 index NULL PRIMARY 4 NULL 1 Using index explain SELECT distinct a from t3 order by a desc limit 2; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t3 index NULL a 5 NULL 40 Using index +1 SIMPLE t3 index NULL a 5 NULL 2 Using index explain SELECT distinct a,b from t3 order by a+1; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t3 ALL NULL NULL NULL NULL 204 Using temporary; Using filesort === modified file 'mysql-test/suite/pbxt/r/func_group.result' --- a/mysql-test/suite/pbxt/r/func_group.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/func_group.result 2009-08-17 15:57:58 +0000 @@ -61,7 +61,7 @@ grp sum NULL NULL 1 7 2 20.25 -3 45.483163247594 +3 45.4831632475944 create table t2 (grp int, a bigint unsigned, c char(10)); insert into t2 select grp,max(a)+max(grp),max(c) from t1 group by grp; replace into t2 select grp, a, c from t1 limit 2,1; @@ -613,8 +613,8 @@ id select_type table type possible_keys explain select max(t1.a3), min(t2.a2) from t1, t2 where t1.a2 = 2 and t1.a3 < 'MIN' and t2.a3 > 'CA'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 range k1 k1 3 NULL 1 Using where; Using index -1 SIMPLE t1 range k1 k1 7 NULL 1 Using where; Using index; Using join buffer +1 SIMPLE t1 range k1 k1 7 NULL 1 Using where; Using index +1 SIMPLE t2 range k1 k1 3 NULL 1 Using where; Using index; Using join buffer explain select min(a4 - 0.01) from t1; id select_type table type possible_keys key key_len ref rows Extra @@ -1186,7 +1186,7 @@ std(s1/s2) 0.21325764 select std(o1/o2) from bug22555; std(o1/o2) -0.21325763586649 +0.213257635866493 select std(e1/e2) from bug22555; std(e1/e2) 0.21325764 @@ -1212,7 +1212,7 @@ round(std(s1/s2), 17) 0.21325763586649341 select std(o1/o2) from bug22555; std(o1/o2) -0.21325763586649 +0.213257635866493 select round(std(e1/e2), 17) from bug22555; round(std(e1/e2), 17) 0.21325763586649341 @@ -1237,7 +1237,7 @@ round(std(s1/s2), 17) 0.21325763586649341 select std(o1/o2) from bug22555; std(o1/o2) -0.21325763586649 +0.213257635866493 select round(std(e1/e2), 17) from bug22555; round(std(e1/e2), 17) 0.21325763586649341 === modified file 'mysql-test/suite/pbxt/r/func_math.result' --- a/mysql-test/suite/pbxt/r/func_math.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/func_math.result 2009-08-17 15:57:58 +0000 @@ -60,7 +60,7 @@ Warnings: Note 1003 select ln(exp(10)) AS `ln(exp(10))`,exp((ln(sqrt(10)) * 2)) AS `exp(ln(sqrt(10))*2)`,ln(-(1)) AS `ln(-1)`,ln(0) AS `ln(0)`,ln(NULL) AS `ln(NULL)` select log2(8),log2(15),log2(-2),log2(0),log2(NULL); log2(8) log2(15) log2(-2) log2(0) log2(NULL) -3 3.9068905956085 NULL NULL NULL +3 3.90689059560852 NULL NULL NULL explain extended select log2(8),log2(15),log2(-2),log2(0),log2(NULL); id select_type table type possible_keys key key_len ref rows filtered Extra 1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL No tables used @@ -68,7 +68,7 @@ Warnings: Note 1003 select log2(8) AS `log2(8)`,log2(15) AS `log2(15)`,log2(-(2)) AS `log2(-2)`,log2(0) AS `log2(0)`,log2(NULL) AS `log2(NULL)` select log10(100),log10(18),log10(-4),log10(0),log10(NULL); log10(100) log10(18) log10(-4) log10(0) log10(NULL) -2 1.2552725051033 NULL NULL NULL +2 1.25527250510331 NULL NULL NULL explain extended select log10(100),log10(18),log10(-4),log10(0),log10(NULL); id select_type table type possible_keys key key_len ref rows filtered Extra 1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL No tables used @@ -85,7 +85,7 @@ Note 1003 select pow(10,log10(10)) AS `p set @@rand_seed1=10000000,@@rand_seed2=1000000; select rand(999999),rand(); rand(999999) rand() -0.014231365187309 0.028870999839968 +0.0142313651873091 0.028870999839968 explain extended select rand(999999),rand(); id select_type table type possible_keys key key_len ref rows filtered Extra 1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL No tables used @@ -101,7 +101,7 @@ Warnings: Note 1003 select pi() AS `pi()`,format(sin((pi() / 2)),6) AS `format(sin(pi()/2),6)`,format(cos((pi() / 2)),6) AS `format(cos(pi()/2),6)`,format(abs(tan(pi())),6) AS `format(abs(tan(pi())),6)`,format((1 / tan(1)),6) AS `format(cot(1),6)`,format(asin(1),6) AS `format(asin(1),6)`,format(acos(0),6) AS `format(acos(0),6)`,format(atan(1),6) AS `format(atan(1),6)` select degrees(pi()),radians(360); degrees(pi()) radians(360) -180 6.2831853071796 +180 6.28318530717959 select format(atan(-2, 2), 6); format(atan(-2, 2), 6) -0.785398 === modified file 'mysql-test/suite/pbxt/r/func_str.result' --- a/mysql-test/suite/pbxt/r/func_str.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/func_str.result 2009-08-17 15:57:58 +0000 @@ -1327,10 +1327,10 @@ cast(rtrim(ltrim(' 20.06 ')) as decimal 20.06 select conv("18383815659218730760",10,10) + 0; conv("18383815659218730760",10,10) + 0 -1.8383815659219e+19 +1.83838156592187e+19 select "18383815659218730760" + 0; "18383815659218730760" + 0 -1.8383815659219e+19 +1.83838156592187e+19 CREATE TABLE t1 (code varchar(10)); INSERT INTO t1 VALUES ('a12'), ('A12'), ('a13'); SELECT ASCII(code), code FROM t1 WHERE code='A12'; === modified file 'mysql-test/suite/pbxt/r/grant.result' --- a/mysql-test/suite/pbxt/r/grant.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/grant.result 2009-08-17 15:57:58 +0000 @@ -457,7 +457,7 @@ Privilege Context Comment Alter Tables To alter the table Alter routine Functions,Procedures To alter or drop stored functions/procedures Create Databases,Tables,Indexes To create new databases and tables -Create routine Functions,Procedures To use CREATE FUNCTION/PROCEDURE +Create routine Databases To use CREATE FUNCTION/PROCEDURE Create temporary tables Databases To use CREATE TEMPORARY TABLE Create view Tables To create new views Create user Server Admin To create new users === modified file 'mysql-test/suite/pbxt/r/group_min_max.result' --- a/mysql-test/suite/pbxt/r/group_min_max.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/group_min_max.result 2009-08-17 15:57:58 +0000 @@ -133,34 +133,34 @@ Table Op Msg_type Msg_text test.t3 analyze status OK explain select a1, min(a2) from t1 group by a1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 130 NULL 10 Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 130 NULL 129 Using index for group-by explain select a1, max(a2) from t1 group by a1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 65 NULL 10 Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 65 NULL 129 Using index for group-by explain select a1, min(a2), max(a2) from t1 group by a1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 130 NULL 10 Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 130 NULL 129 Using index for group-by explain select a1, a2, b, min(c), max(c) from t1 group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using index for group-by explain select a1,a2,b,max(c),min(c) from t1 group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using index for group-by explain select a1,a2,b,max(c),min(c) from t2 group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t2 range NULL idx_t2_1 # NULL # Using index for group-by explain select min(a2), a1, max(a2), min(a2), a1 from t1 group by a1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 130 NULL 10 Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 130 NULL 129 Using index for group-by explain select a1, b, min(c), a1, max(c), b, a2, max(c), max(c) from t1 group by a1, a2, b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using index for group-by explain select min(a2) from t1 group by a1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 130 NULL 10 Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 130 NULL 129 Using index for group-by explain select a2, min(c), max(c) from t1 group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using index for group-by select a1, min(a2) from t1 group by a1; a1 min(a2) a a @@ -293,13 +293,13 @@ id select_type table type possible_keys 1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_2 65 NULL 1 Using where explain select a1,a2,b, max(c) from t1 where a1 >= 'c' or a1 < 'b' group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_1 147 NULL 1 Using where; Using index for group-by +1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_1 147 NULL 2 Using where; Using index for group-by explain select a1, max(c) from t1 where a1 >= 'c' or a1 < 'b' group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_1 147 NULL 1 Using where; Using index for group-by +1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_1 147 NULL 2 Using where; Using index for group-by explain select a1,a2,b,min(c),max(c) from t1 where a1 >= 'c' or a2 < 'b' group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_1 147 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_1 147 NULL 129 Using where; Using index for group-by explain select a1,a2,b, max(c) from t1 where a1 = 'z' or a1 = 'b' or a1 = 'd' group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_1 65 NULL 3 Using where; Using index @@ -669,40 +669,40 @@ d l421 d p422 explain select a1,a2,b,max(c),min(c) from t1 where (a2 = 'a') and (b = 'b') group by a1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using where; Using index for group-by explain select a1,max(c),min(c) from t1 where (a2 = 'a') and (b = 'b') group by a1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using where; Using index for group-by explain select a1,a2,b, max(c) from t1 where (b = 'b') group by a1,a2; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using where; Using index for group-by explain select a1,a2,b,min(c),max(c) from t1 where (b = 'b') group by a1,a2; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using where; Using index for group-by explain select a1,a2, max(c) from t1 where (b = 'b') group by a1,a2; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using where; Using index for group-by explain select a1,a2,b,max(c),min(c) from t2 where (a2 = 'a') and (b = 'b') group by a1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 range NULL idx_t2_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t2 range NULL idx_t2_1 163 NULL 165 Using where; Using index for group-by explain select a1,max(c),min(c) from t2 where (a2 = 'a') and (b = 'b') group by a1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 range NULL idx_t2_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t2 range NULL idx_t2_1 163 NULL 165 Using where; Using index for group-by explain select a1,a2,b, max(c) from t2 where (b = 'b') group by a1,a2; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 range NULL idx_t2_1 146 NULL 10 Using where; Using index for group-by +1 SIMPLE t2 range NULL idx_t2_1 146 NULL 165 Using where; Using index for group-by explain select a1,a2,b,min(c),max(c) from t2 where (b = 'b') group by a1,a2; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 range NULL idx_t2_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t2 range NULL idx_t2_1 163 NULL 165 Using where; Using index for group-by explain select a1,a2, max(c) from t2 where (b = 'b') group by a1,a2; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 range NULL idx_t2_1 146 NULL 10 Using where; Using index for group-by +1 SIMPLE t2 range NULL idx_t2_1 146 NULL 165 Using where; Using index for group-by explain select a1,a2,b,max(c),min(c) from t3 where (a2 = 'a') and (b = 'b') group by a1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t3 range NULL idx_t3_1 6 NULL 10 Using where; Using index for group-by +1 SIMPLE t3 range NULL idx_t3_1 6 NULL 193 Using where; Using index for group-by explain select a1,max(c),min(c) from t3 where (a2 = 'a') and (b = 'b') group by a1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t3 range NULL idx_t3_1 6 NULL 10 Using where; Using index for group-by +1 SIMPLE t3 range NULL idx_t3_1 6 NULL 193 Using where; Using index for group-by select a1,a2,b,max(c),min(c) from t1 where (a2 = 'a') and (b = 'b') group by a1; a1 a2 b max(c) min(c) a a b h112 e112 @@ -804,22 +804,22 @@ b h212 e212 c h312 e312 explain select a1,a2,b,min(c) from t2 where (a2 = 'a') and b is NULL group by a1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 range NULL idx_t2_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t2 range NULL idx_t2_1 163 NULL 165 Using where; Using index for group-by explain select a1,a2,b,max(c) from t2 where (a2 = 'a') and b is NULL group by a1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 range NULL idx_t2_1 146 NULL 10 Using where; Using index for group-by +1 SIMPLE t2 range NULL idx_t2_1 146 NULL 165 Using where; Using index for group-by explain select a1,a2,b,min(c) from t2 where b is NULL group by a1,a2; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 range NULL idx_t2_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t2 range NULL idx_t2_1 163 NULL 165 Using where; Using index for group-by explain select a1,a2,b,max(c) from t2 where b is NULL group by a1,a2; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 range NULL idx_t2_1 146 NULL 10 Using where; Using index for group-by +1 SIMPLE t2 range NULL idx_t2_1 146 NULL 165 Using where; Using index for group-by explain select a1,a2,b,min(c),max(c) from t2 where b is NULL group by a1,a2; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 range NULL idx_t2_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t2 range NULL idx_t2_1 163 NULL 165 Using where; Using index for group-by explain select a1,a2,b,min(c),max(c) from t2 where b is NULL group by a1,a2; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t2 range NULL idx_t2_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t2 range NULL idx_t2_1 163 NULL 165 Using where; Using index for group-by select a1,a2,b,min(c) from t2 where (a2 = 'a') and b is NULL group by a1; a1 a2 b min(c) a a NULL a777 @@ -849,49 +849,49 @@ id select_type table type possible_keys 1 SIMPLE t1 range NULL idx_t1_1 147 NULL # Using where; Using index for group-by explain select a1,a2,b,min(c),max(c) from t1 where (c > 'b1') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 163 NULL 129 Using where; Using index for group-by explain select a1,a2,b, max(c) from t1 where (c > 'f123') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using where; Using index for group-by explain select a1,a2,b,min(c),max(c) from t1 where (c > 'f123') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 163 NULL 129 Using where; Using index for group-by explain select a1,a2,b, max(c) from t1 where (c < 'a0') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 163 NULL 129 Using where; Using index for group-by explain select a1,a2,b,min(c),max(c) from t1 where (c < 'a0') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 163 NULL 129 Using where; Using index for group-by explain select a1,a2,b, max(c) from t1 where (c < 'k321') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 163 NULL 129 Using where; Using index for group-by explain select a1,a2,b,min(c),max(c) from t1 where (c < 'k321') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 163 NULL 129 Using where; Using index for group-by explain select a1,a2,b, max(c) from t1 where (c < 'a0') or (c > 'b1') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 163 NULL 129 Using where; Using index for group-by explain select a1,a2,b,min(c),max(c) from t1 where (c < 'a0') or (c > 'b1') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 163 NULL 129 Using where; Using index for group-by explain select a1,a2,b, max(c) from t1 where (c > 'b1') or (c <= 'g1') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using where; Using index for group-by explain select a1,a2,b,min(c),max(c) from t1 where (c > 'b1') or (c <= 'g1') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using where; Using index for group-by explain select a1,a2,b,min(c),max(c) from t1 where (c > 'b111') and (c <= 'g112') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 163 NULL 129 Using where; Using index for group-by explain select a1,a2,b,min(c),max(c) from t1 where (c < 'c5') or (c = 'g412') or (c = 'k421') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 163 NULL 129 Using where; Using index for group-by explain select a1,a2,b,min(c),max(c) from t1 where ((c > 'b111') and (c <= 'g112')) or ((c > 'd000') and (c <= 'i110')) group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 163 NULL 129 Using where; Using index for group-by explain select a1,a2,b,min(c),max(c) from t1 where (c between 'b111' and 'g112') or (c between 'd000' and 'i110') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 163 NULL 129 Using where; Using index for group-by explain select a1,a2,b, max(c) from t2 where (c > 'b1') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t2 range NULL idx_t2_1 146 NULL # Using where; Using index for group-by @@ -1364,29 +1364,29 @@ explain select a1,a2,b,min(c),max(c) fro where exists ( select * from t2 where t2.c > 'b1' ) group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY t1 range NULL idx_t1_1 147 NULL 10 Using index for group-by +1 PRIMARY t1 range NULL idx_t1_1 147 NULL 129 Using index for group-by 2 SUBQUERY t2 index NULL idx_t2_1 163 NULL 164 Using where; Using index explain select a1,a2,b,min(c),max(c) from t1 where (a1 >= 'c' or a2 < 'b') and (b > 'a') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_1 147 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_1 147 NULL 129 Using where; Using index for group-by explain select a1,a2,b,min(c),max(c) from t1 where (a1 >= 'c' or a2 < 'b') and (c > 'b111') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_1 163 NULL 129 Using where; Using index for group-by explain select a1,a2,b,min(c),max(c) from t1 where (a2 >= 'b') and (b = 'a') and (c > 'b111') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 163 NULL 129 Using where; Using index for group-by explain select a1,a2,b,min(c) from t1 where ((a1 > 'a') or (a1 < '9')) and ((a2 >= 'b') and (a2 < 'z')) and (b = 'a') and ((c < 'h112') or (c = 'j121') or (c > 'k121' and c < 'm122') or (c > 'o122')) group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_1 163 NULL 1 Using where; Using index for group-by +1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_1 163 NULL 2 Using where; Using index for group-by explain select a1,a2,b,min(c) from t1 where ((a1 > 'a') or (a1 < '9')) and ((a2 >= 'b') and (a2 < 'z')) and (b = 'a') and ((c = 'j121') or (c > 'k121' and c < 'm122') or (c > 'o122') or (c < 'h112') or (c = 'c111')) group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_1 163 NULL 1 Using where; Using index for group-by +1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_1 163 NULL 2 Using where; Using index for group-by explain select a1,a2,b,min(c) from t1 where (a1 > 'a') and (a2 > 'a') and (b = 'c') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_2 65 NULL 1 Using where explain select a1,a2,b,min(c) from t1 where (ord(a1) > 97) and (ord(a2) + ord(a1) > 194) and (b = 'c') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using where; Using index for group-by explain select a1,a2,b,min(c),max(c) from t2 where (a1 >= 'c' or a2 < 'b') and (b > 'a') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t2 range idx_t2_0,idx_t2_1,idx_t2_2 idx_t2_1 163 NULL # Using where; Using index for group-by @@ -1491,13 +1491,13 @@ select a1,a2,b,min(c) from t2 where (a1 a1 a2 b min(c) explain select a1,a2,b from t1 where (a1 >= 'c' or a2 < 'b') and (b > 'a') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_1 147 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_1 147 NULL 129 Using where; Using index for group-by explain select a1,a2,b from t1 where (a2 >= 'b') and (b = 'a') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using where; Using index for group-by explain select a1,a2,b,c from t1 where (a2 >= 'b') and (b = 'a') and (c = 'i121') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 163 NULL 129 Using where; Using index for group-by explain select a1,a2,b from t1 where (a1 > 'a') and (a2 > 'a') and (b = 'c') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_2 147 NULL 1 Using where @@ -1554,13 +1554,13 @@ select a1,a2,b from t2 where (a1 > 'a') a1 a2 b explain select distinct a1,a2,b from t1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using index for group-by explain select distinct a1,a2,b from t1 where (a2 >= 'b') and (b = 'a'); id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using where; Using index for group-by explain extended select distinct a1,a2,b,c from t1 where (a2 >= 'b') and (b = 'a') and (c = 'i121'); id select_type table type possible_keys key key_len ref rows filtered Extra -1 SIMPLE t1 range NULL idx_t1_1 163 NULL 10 100.00 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 163 NULL 129 99.22 Using where; Using index for group-by Warnings: Note 1003 select distinct `test`.`t1`.`a1` AS `a1`,`test`.`t1`.`a2` AS `a2`,`test`.`t1`.`b` AS `b`,`test`.`t1`.`c` AS `c` from `test`.`t1` where ((`test`.`t1`.`c` = 'i121') and (`test`.`t1`.`b` = 'a') and (`test`.`t1`.`a2` >= 'b')) explain select distinct a1,a2,b from t1 where (a1 > 'a') and (a2 > 'a') and (b = 'c'); @@ -1577,7 +1577,7 @@ id select_type table type possible_keys 1 SIMPLE t2 range NULL idx_t2_2 146 NULL # Using where; Using index for group-by explain extended select distinct a1,a2,b,c from t2 where (a2 >= 'b') and (b = 'a') and (c = 'i121'); id select_type table type possible_keys key key_len ref rows filtered Extra -1 SIMPLE t2 range NULL idx_t2_1 163 NULL 10 100.00 Using where; Using index for group-by +1 SIMPLE t2 range NULL idx_t2_1 163 NULL 165 99.39 Using where; Using index for group-by Warnings: Note 1003 select distinct `test`.`t2`.`a1` AS `a1`,`test`.`t2`.`a2` AS `a2`,`test`.`t2`.`b` AS `b`,`test`.`t2`.`c` AS `c` from `test`.`t2` where ((`test`.`t2`.`c` = 'i121') and (`test`.`t2`.`b` = 'a') and (`test`.`t2`.`a2` >= 'b')) explain select distinct a1,a2,b from t2 where (a1 > 'a') and (a2 > 'a') and (b = 'c'); @@ -1702,19 +1702,19 @@ c e d e explain select distinct a1,a2,b from t1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using index for group-by explain select distinct a1,a2,b from t1 where (a2 >= 'b') and (b = 'a') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using where; Using index for group-by explain select distinct a1,a2,b,c from t1 where (a2 >= 'b') and (b = 'a') and (c = 'i121') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 163 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 163 NULL 129 Using where; Using index for group-by explain select distinct a1,a2,b from t1 where (a1 > 'a') and (a2 > 'a') and (b = 'c') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_2 147 NULL 1 Using where explain select distinct b from t1 where (a2 >= 'b') and (b = 'a') group by a1,a2,b; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using where; Using index for group-by; Using temporary; Using filesort +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using where; Using index for group-by; Using temporary; Using filesort explain select distinct a1,a2,b from t2; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t2 range NULL idx_t2_2 146 NULL # Using index for group-by @@ -1846,7 +1846,7 @@ id select_type table type possible_keys 1 SIMPLE t1 range idx_t1_0,idx_t1_1,idx_t1_2 idx_t1_2 65 NULL 1 Using where explain select concat(ord(min(b)),ord(max(b))),min(b),max(b) from t1 group by a1,a2; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 147 NULL 10 Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 147 NULL 129 Using index for group-by select a1,a2,b, concat(min(c), max(c)) from t1 where a1 < 'd' group by a1,a2,b; a1 a2 b concat(min(c), max(c)) a a a a111d111 @@ -1985,7 +1985,7 @@ c d explain select a1 from t1 where a2 = 'b' group by a1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 130 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 130 NULL 129 Using where; Using index for group-by select a1 from t1 where a2 = 'b' group by a1; a1 a @@ -1994,7 +1994,7 @@ c d explain select distinct a1 from t1 where a2 = 'b'; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx_t1_1 130 NULL 10 Using where; Using index for group-by +1 SIMPLE t1 range NULL idx_t1_1 130 NULL 129 Using where; Using index for group-by select distinct a1 from t1 where a2 = 'b'; a1 a @@ -2188,7 +2188,7 @@ INSERT INTO t1 (a, b) VALUES (1,1), (1,2 (2,2), (2,3), (2,1), (3,1), (4,1), (4,2), (4,3), (4,4), (4,5), (4,6); EXPLAIN SELECT max(b), a FROM t1 GROUP BY a; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL a 5 NULL 8 Using index for group-by +1 SIMPLE t1 index NULL a 10 NULL 15 Using index FLUSH STATUS; SELECT max(b), a FROM t1 GROUP BY a; max(b) a @@ -2202,7 +2202,7 @@ Handler_read_key 0 Handler_read_next 0 EXPLAIN SELECT max(b), a FROM t1 GROUP BY a; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL a 5 NULL 8 Using index for group-by +1 SIMPLE t1 index NULL a 10 NULL 15 Using index FLUSH STATUS; CREATE TABLE t2 SELECT max(b), a FROM t1 GROUP BY a; SHOW STATUS LIKE 'handler_read__e%'; @@ -2235,14 +2235,14 @@ Handler_read_next 0 EXPLAIN (SELECT max(b), a FROM t1 GROUP BY a) UNION (SELECT max(b), a FROM t1 GROUP BY a); id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY t1 range NULL a 5 NULL 8 Using index for group-by -2 UNION t1 range NULL a 5 NULL 8 Using index for group-by +1 PRIMARY t1 index NULL a 10 NULL 15 Using index +2 UNION t1 index NULL a 10 NULL 15 Using index NULL UNION RESULT <union1,2> ALL NULL NULL NULL NULL NULL EXPLAIN SELECT (SELECT max(b) FROM t1 GROUP BY a HAVING a < 2) x FROM t1 AS t1_outer; id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1_outer index NULL a 10 NULL 15 Using index -2 SUBQUERY t1 range NULL a 5 NULL 8 Using index for group-by +2 SUBQUERY t1 index NULL a 10 NULL 15 Using index EXPLAIN SELECT 1 FROM t1 AS t1_outer WHERE EXISTS (SELECT max(b) FROM t1 GROUP BY a HAVING a < 2); id select_type table type possible_keys key key_len ref rows Extra @@ -2252,7 +2252,7 @@ EXPLAIN SELECT 1 FROM t1 AS t1_outer WHE (SELECT max(b) FROM t1 GROUP BY a HAVING a < 2) > 12; id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY NULL NULL NULL NULL NULL NULL NULL Impossible WHERE -2 SUBQUERY t1 range NULL a 5 NULL 8 Using index for group-by +2 SUBQUERY t1 index NULL a 10 NULL 15 Using index EXPLAIN SELECT 1 FROM t1 AS t1_outer WHERE a IN (SELECT max(b) FROM t1 GROUP BY a HAVING a < 2); id select_type table type possible_keys key key_len ref rows Extra @@ -2261,21 +2261,21 @@ id select_type table type possible_keys EXPLAIN SELECT 1 FROM t1 AS t1_outer GROUP BY a HAVING a > (SELECT max(b) FROM t1 GROUP BY a HAVING a < 2); id select_type table type possible_keys key key_len ref rows Extra -1 PRIMARY t1_outer range NULL a 5 NULL 8 Using index for group-by -2 SUBQUERY t1 range NULL a 5 NULL 8 Using index for group-by +1 PRIMARY t1_outer index NULL a 10 NULL 15 Using index +2 SUBQUERY t1 index NULL a 10 NULL 15 Using index EXPLAIN SELECT 1 FROM t1 AS t1_outer1 JOIN t1 AS t1_outer2 ON t1_outer1.a = (SELECT max(b) FROM t1 GROUP BY a HAVING a < 2) AND t1_outer1.b = t1_outer2.b; id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1_outer1 ref a a 5 const 1 Using where; Using index 1 PRIMARY t1_outer2 index NULL a 10 NULL 15 Using where; Using index; Using join buffer -2 SUBQUERY t1 range NULL a 5 NULL 8 Using index for group-by +2 SUBQUERY t1 index NULL a 10 NULL 15 Using index EXPLAIN SELECT (SELECT (SELECT max(b) FROM t1 GROUP BY a HAVING a < 2) x FROM t1 AS t1_outer) x2 FROM t1 AS t1_outer2; id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1_outer2 index NULL a 10 NULL 15 Using index 2 SUBQUERY t1_outer index NULL a 10 NULL 15 Using index -3 SUBQUERY t1 range NULL a 5 NULL 8 Using index for group-by +3 SUBQUERY t1 index NULL a 10 NULL 15 Using index CREATE TABLE t3 LIKE t1; FLUSH STATUS; INSERT INTO t3 SELECT a,MAX(b) FROM t1 GROUP BY a; @@ -2312,7 +2312,7 @@ INSERT INTO t1 VALUES (4), (2), (1), (2), (2), (4), (1), (4); EXPLAIN SELECT DISTINCT(a) FROM t1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx 5 NULL 9 Using index for group-by +1 SIMPLE t1 index NULL idx 5 NULL 16 Using index SELECT DISTINCT(a) FROM t1; a 1 @@ -2320,7 +2320,7 @@ a 4 EXPLAIN SELECT SQL_BIG_RESULT DISTINCT(a) FROM t1; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL idx 5 NULL 9 Using index for group-by +1 SIMPLE t1 index NULL idx 5 NULL 16 Using index SELECT SQL_BIG_RESULT DISTINCT(a) FROM t1; a 1 @@ -2345,7 +2345,7 @@ CREATE INDEX break_it ON t1 (a, b); EXPLAIN SELECT a, MIN(b), MAX(b) FROM t1 GROUP BY a ORDER BY a; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL break_it 10 NULL 7 Using index for group-by +1 SIMPLE t1 index NULL break_it 10 NULL 12 Using index SELECT a, MIN(b), MAX(b) FROM t1 GROUP BY a ORDER BY a; a MIN(b) MAX(b) 1 1 3 @@ -2355,7 +2355,7 @@ a MIN(b) MAX(b) EXPLAIN SELECT a, MIN(b), MAX(b) FROM t1 GROUP BY a ORDER BY a DESC; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 range NULL break_it 10 NULL 7 Using index for group-by; Using temporary; Using filesort +1 SIMPLE t1 index NULL break_it 10 NULL 12 Using index SELECT a, MIN(b), MAX(b) FROM t1 GROUP BY a ORDER BY a DESC; a MIN(b) MAX(b) 4 1 3 === modified file 'mysql-test/suite/pbxt/r/join.result' --- a/mysql-test/suite/pbxt/r/join.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/join.result 2009-08-17 15:57:58 +0000 @@ -774,7 +774,7 @@ insert into t3 select * from t2 where a explain select * from t2,t3 where t2.a < 200 and t2.b=t3.b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t2 range a,b a 5 NULL 1 Using where -1 SIMPLE t3 ref b b 5 test.t2.b 11 Using where +1 SIMPLE t3 ref b b 5 test.t2.b 1 Using where drop table t1, t2, t3; create table t1 (a int); insert into t1 values (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); === modified file 'mysql-test/suite/pbxt/r/join_nested.result' --- a/mysql-test/suite/pbxt/r/join_nested.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/join_nested.result 2009-08-17 15:57:58 +0000 @@ -851,7 +851,7 @@ ON t3.a=1 AND t3.b=t2.b AND t2.b=t4.b; id select_type table type possible_keys key key_len ref rows filtered Extra 1 SIMPLE t3 ALL NULL NULL NULL NULL 2 100.00 1 SIMPLE t4 ALL NULL NULL NULL NULL 2 100.00 Using join buffer -1 SIMPLE t2 ref idx_b idx_b 5 test.t3.b 2 100.00 +1 SIMPLE t2 ref idx_b idx_b 5 test.t3.b 1 100.00 1 SIMPLE t1 ALL NULL NULL NULL NULL 3 100.00 Warnings: Note 1003 select `test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b`,`test`.`t4`.`a` AS `a`,`test`.`t4`.`b` AS `b` from `test`.`t3` join `test`.`t4` left join (`test`.`t1` join `test`.`t2`) on(((`test`.`t3`.`a` = 1) and (`test`.`t3`.`b` = `test`.`t2`.`b`) and (`test`.`t2`.`b` = `test`.`t4`.`b`))) where 1 @@ -958,15 +958,15 @@ id select_type table type possible_keys 1 SIMPLE t0 ALL NULL NULL NULL NULL 3 100.00 Using where 1 SIMPLE t1 ALL NULL NULL NULL NULL 3 100.00 Using where; Using join buffer 1 SIMPLE t2 ALL NULL NULL NULL NULL 3 100.00 Using where +1 SIMPLE t4 ref idx_b idx_b 5 test.t2.b 1 100.00 1 SIMPLE t3 ALL NULL NULL NULL NULL 2 100.00 Using where -1 SIMPLE t4 ref idx_b idx_b 5 test.t2.b 2 100.00 Using where 1 SIMPLE t5 ALL idx_b NULL NULL NULL 3 100.00 Using where 1 SIMPLE t7 ALL NULL NULL NULL NULL 2 100.00 Using where 1 SIMPLE t6 ALL NULL NULL NULL NULL 3 100.00 Using where 1 SIMPLE t8 ALL NULL NULL NULL NULL 2 100.00 Using where 1 SIMPLE t9 ALL NULL NULL NULL NULL 3 100.00 Using where; Using join buffer Warnings: -Note 1003 select `test`.`t0`.`a` AS `a`,`test`.`t0`.`b` AS `b`,`test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b`,`test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b`,`test`.`t4`.`a` AS `a`,`test`.`t4`.`b` AS `b`,`test`.`t5`.`a` AS `a`,`test`.`t5`.`b` AS `b`,`test`.`t6`.`a` AS `a`,`test`.`t6`.`b` AS `b`,`test`.`t7`.`a` AS `a`,`test`.`t7`.`b` AS `b`,`test`.`t8`.`a` AS `a`,`test`.`t8`.`b` AS `b`,`test`.`t9`.`a` AS `a`,`test`.`t9`.`b` AS `b` from `test`.`t0` join `test`.`t1` left join (`test`.`t2` left join (`test`.`t3` join `test`.`t4`) on(((`test`.`t4`.`b` = `test`.`t2`.`b`) and (`test`.`t3`.`a` = 1))) join `test`.`t5` left join (`test`.`t6` join `test`.`t7` left join `test`.`t8` on(((`test`.`t8`.`b` = `test`.`t5`.`b`) and (`test`.`t6`.`b` < 10)))) on(((`test`.`t7`.`b` = `test`.`t5`.`b`) and (`test`.`t6`.`b` >= 2)))) on((((`test`.`t3`.`b` = 2) or isnull(`test`.`t3`.`c`)) and ((`test`.`t6`.`b` = 2) or isnull(`test`.`t6`.`c`)) and ((`test`.`t5`.`b` = `test`.`t0`.`b`) or isnull(`test`.`t3`.`c`) or isnull(`test`.`t6`.`c`) or isnull(`test`.`t8`.`c`)) and (`test`.`t1`.`a` <> 2))) join `test`.`t9` where ((`test`.`t9`.`a` = 1) and (`test`.`t1`.`b` = `test`.`t0`.`b`) and (`test`.`t0`.`a` = 1) and ((`test`.`t2`.`a` >= 4) or isnull(`test`.`t2`.`c`)) and ((`test`.`t3`.`a` < 5) or isnull(`test`.`t3`.`c`)) and ((`test`.`t4`.`b` = `test`.`t3`.`b`) or isnull(`test`.`t3`.`c`) or isnull(`test`.`t4`.`c`)) and ((`test`.`t5`.`a` >= 2) or isnull(`test`.`t5`.`c`)) and ((`test`.`t6`.`a` >= 4) or isnull(`test`.`t6`.`c`)) and ((`test`.`t7`.`a` <= 2) or isnull(`test`.`t7`.`c`)) and ((`test`.`t8`.`a` < 1) or isnull(`test`.`t8`.`c`)) and ((`test`.`t9`.`b` = `test`.`t8`.`b`) or isnull(`test`.`t8`.`c`))) +Note 1003 select `test`.`t0`.`a` AS `a`,`test`.`t0`.`b` AS `b`,`test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b`,`test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b`,`test`.`t4`.`a` AS `a`,`test`.`t4`.`b` AS `b`,`test`.`t5`.`a` AS `a`,`test`.`t5`.`b` AS `b`,`test`.`t6`.`a` AS `a`,`test`.`t6`.`b` AS `b`,`test`.`t7`.`a` AS `a`,`test`.`t7`.`b` AS `b`,`test`.`t8`.`a` AS `a`,`test`.`t8`.`b` AS `b`,`test`.`t9`.`a` AS `a`,`test`.`t9`.`b` AS `b` from `test`.`t0` join `test`.`t1` left join (`test`.`t2` left join (`test`.`t3` join `test`.`t4`) on(((`test`.`t4`.`b` = `test`.`t2`.`b`) and (`test`.`t3`.`a` = 1))) join `test`.`t5` left join (`test`.`t6` join `test`.`t7` left join `test`.`t8` on(((`test`.`t8`.`b` = `test`.`t5`.`b`) and (`test`.`t6`.`b` < 10)))) on(((`test`.`t7`.`b` = `test`.`t5`.`b`) and (`test`.`t6`.`b` >= 2)))) on((((`test`.`t3`.`b` = 2) or isnull(`test`.`t3`.`c`)) and ((`test`.`t6`.`b` = 2) or isnull(`test`.`t6`.`c`)) and ((`test`.`t5`.`b` = `test`.`t0`.`b`) or isnull(`test`.`t3`.`c`) or isnull(`test`.`t6`.`c`) or isnull(`test`.`t8`.`c`)) and (`test`.`t1`.`a` <> 2))) join `test`.`t9` where ((`test`.`t9`.`a` = 1) and (`test`.`t1`.`b` = `test`.`t0`.`b`) and (`test`.`t0`.`a` = 1) and ((`test`.`t2`.`a` >= 4) or isnull(`test`.`t2`.`c`)) and ((`test`.`t3`.`a` < 5) or isnull(`test`.`t3`.`c`)) and ((`test`.`t3`.`b` = `test`.`t4`.`b`) or isnull(`test`.`t3`.`c`) or isnull(`test`.`t4`.`c`)) and ((`test`.`t5`.`a` >= 2) or isnull(`test`.`t5`.`c`)) and ((`test`.`t6`.`a` >= 4) or isnull(`test`.`t6`.`c`)) and ((`test`.`t7`.`a` <= 2) or isnull(`test`.`t7`.`c`)) and ((`test`.`t8`.`a` < 1) or isnull(`test`.`t8`.`c`)) and ((`test`.`t9`.`b` = `test`.`t8`.`b`) or isnull(`test`.`t8`.`c`))) CREATE INDEX idx_b ON t8(b); EXPLAIN EXTENDED SELECT t0.a,t0.b,t1.a,t1.b,t2.a,t2.b,t3.a,t3.b,t4.a,t4.b, @@ -1008,14 +1008,14 @@ id select_type table type possible_keys 1 SIMPLE t1 ALL NULL NULL NULL NULL 3 100.00 Using where; Using join buffer 1 SIMPLE t2 ALL NULL NULL NULL NULL 3 100.00 Using where 1 SIMPLE t3 ALL NULL NULL NULL NULL 2 100.00 Using where -1 SIMPLE t4 ref idx_b idx_b 5 test.t2.b 2 100.00 Using where +1 SIMPLE t4 ref idx_b idx_b 5 test.t2.b 1 100.00 1 SIMPLE t5 ALL idx_b NULL NULL NULL 3 100.00 Using where -1 SIMPLE t7 ALL NULL NULL NULL NULL 2 100.00 Using where 1 SIMPLE t6 ALL NULL NULL NULL NULL 3 100.00 Using where -1 SIMPLE t8 ref idx_b idx_b 5 test.t5.b 2 100.00 Using where +1 SIMPLE t7 ALL NULL NULL NULL NULL 2 100.00 Using where +1 SIMPLE t8 ref idx_b idx_b 5 test.t5.b 1 100.00 Using where 1 SIMPLE t9 ALL NULL NULL NULL NULL 3 100.00 Using where; Using join buffer +Note 1003 select `test`.`t0`.`a` AS `a`,`test`.`t0`.`b` AS `b`,`test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b`,`test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b`,`test`.`t4`.`a` AS `a`,`test`.`t4`.`b` AS `b`,`test`.`t5`.`a` AS `a`,`test`.`t5`.`b` AS `b`,`test`.`t6`.`a` AS `a`,`test`.`t6`.`b` AS `b`,`test`.`t7`.`a` AS `a`,`test`.`t7`.`b` AS `b`,`test`.`t8`.`a` AS `a`,`test`.`t8`.`b` AS `b`,`test`.`t9`.`a` AS `a`,`test`.`t9`.`b` AS `b` from `test`.`t0` join `test`.`t1` left join (`test`.`t2` left join (`test`.`t3` join `test`.`t4`) on(((`test`.`t4`.`b` = `test`.`t2`.`b`) and (`test`.`t3`.`a` = 1))) join `test`.`t5` left join (`test`.`t6` join `test`.`t7` left join `test`.`t8` on(((`test`.`t8`.`b` = `test`.`t5`.`b`) and (`test`.`t6`.`b` < 10)))) on(((`test`.`t7`.`b` = `test`.`t5`.`b`) and (`test`.`t6`.`b` >= 2)))) on((((`test`.`t3`.`b` = 2) or isnull(`test`.`t3`.`c`)) and ((`test`.`t6`.`b` = 2) or isnull(`test`.`t6`.`c`)) and ((`test`.`t5`.`b` = `test`.`t0`.`b`) or isnull(`test`.`t3`.`c`) or isnull(`test`.`t6`.`c`) or isnull(`test`.`t8`.`c`)) and (`test`.`t1`.`a` <> 2))) join `test`.`t9` where ((`test`.`t9`.`a` = 1) and (`test`.`t1`.`b` = `test`.`t0`.`b`) and (`test`.`t0`.`a` = 1) and ((`test`.`t2`.`a` >= 4) or isnull(`test`.`t2`.`c`)) and ((`test`.`t3`.`a` < 5) or isnull(`test`.`t3`.`c`)) and ((`test`.`t3`.`b` = `test`.`t4`.`b`) or isnull(`test`.`t3`.`c`) or isnull(`test`.`t4`.`c`)) and ((`test`.`t5`.`a` >= 2) or isnull(`test`.`t5`.`c`)) and ((`test`.`t6`.`a` >= 4) or isnull(`test`.`t6`.`c`)) and ((`test`.`t7`.`a` <= 2) or isnull(`test`.`t7`.`c`)) and ((`test`.`t8`.`a` < 1) or isnull(`test`.`t8`.`c`)) and ((`test`.`t9`.`b` = `test`.`t8`.`b`) or isnull(`test`.`t8`.`c`))) Warnings: -Note 1003 select `test`.`t0`.`a` AS `a`,`test`.`t0`.`b` AS `b`,`test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b`,`test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b`,`test`.`t4`.`a` AS `a`,`test`.`t4`.`b` AS `b`,`test`.`t5`.`a` AS `a`,`test`.`t5`.`b` AS `b`,`test`.`t6`.`a` AS `a`,`test`.`t6`.`b` AS `b`,`test`.`t7`.`a` AS `a`,`test`.`t7`.`b` AS `b`,`test`.`t8`.`a` AS `a`,`test`.`t8`.`b` AS `b`,`test`.`t9`.`a` AS `a`,`test`.`t9`.`b` AS `b` from `test`.`t0` join `test`.`t1` left join (`test`.`t2` left join (`test`.`t3` join `test`.`t4`) on(((`test`.`t4`.`b` = `test`.`t2`.`b`) and (`test`.`t3`.`a` = 1))) join `test`.`t5` left join (`test`.`t6` join `test`.`t7` left join `test`.`t8` on(((`test`.`t8`.`b` = `test`.`t5`.`b`) and (`test`.`t6`.`b` < 10)))) on(((`test`.`t7`.`b` = `test`.`t5`.`b`) and (`test`.`t6`.`b` >= 2)))) on((((`test`.`t3`.`b` = 2) or isnull(`test`.`t3`.`c`)) and ((`test`.`t6`.`b` = 2) or isnull(`test`.`t6`.`c`)) and ((`test`.`t5`.`b` = `test`.`t0`.`b`) or isnull(`test`.`t3`.`c`) or isnull(`test`.`t6`.`c`) or isnull(`test`.`t8`.`c`)) and (`test`.`t1`.`a` <> 2))) join `test`.`t9` where ((`test`.`t9`.`a` = 1) and (`test`.`t1`.`b` = `test`.`t0`.`b`) and (`test`.`t0`.`a` = 1) and ((`test`.`t2`.`a` >= 4) or isnull(`test`.`t2`.`c`)) and ((`test`.`t3`.`a` < 5) or isnull(`test`.`t3`.`c`)) and ((`test`.`t4`.`b` = `test`.`t3`.`b`) or isnull(`test`.`t3`.`c`) or isnull(`test`.`t4`.`c`)) and ((`test`.`t5`.`a` >= 2) or isnull(`test`.`t5`.`c`)) and ((`test`.`t6`.`a` >= 4) or isnull(`test`.`t6`.`c`)) and ((`test`.`t7`.`a` <= 2) or isnull(`test`.`t7`.`c`)) and ((`test`.`t8`.`a` < 1) or isnull(`test`.`t8`.`c`)) and ((`test`.`t9`.`b` = `test`.`t8`.`b`) or isnull(`test`.`t8`.`c`))) CREATE INDEX idx_b ON t1(b); CREATE INDEX idx_a ON t0(a); EXPLAIN EXTENDED @@ -1055,17 +1055,17 @@ t0.b=t1.b AND (t9.a=1); id select_type table type possible_keys key key_len ref rows filtered Extra 1 SIMPLE t0 ref idx_a idx_a 5 const 1 100.00 Using where -1 SIMPLE t1 ref idx_b idx_b 5 test.t0.b 2 100.00 Using where +1 SIMPLE t1 ref idx_b idx_b 5 test.t0.b 1 100.00 Using where 1 SIMPLE t2 ALL NULL NULL NULL NULL 3 100.00 Using where 1 SIMPLE t3 ALL NULL NULL NULL NULL 2 100.00 Using where -1 SIMPLE t4 ref idx_b idx_b 5 test.t2.b 2 100.00 Using where +1 SIMPLE t4 ref idx_b idx_b 5 test.t2.b 1 100.00 1 SIMPLE t5 ALL idx_b NULL NULL NULL 3 100.00 Using where -1 SIMPLE t7 ALL NULL NULL NULL NULL 2 100.00 Using where 1 SIMPLE t6 ALL NULL NULL NULL NULL 3 100.00 Using where -1 SIMPLE t8 ref idx_b idx_b 5 test.t5.b 2 100.00 Using where +1 SIMPLE t7 ALL NULL NULL NULL NULL 2 100.00 Using where +1 SIMPLE t8 ref idx_b idx_b 5 test.t5.b 1 100.00 Using where 1 SIMPLE t9 ALL NULL NULL NULL NULL 3 100.00 Using where; Using join buffer +Note 1003 select `test`.`t0`.`a` AS `a`,`test`.`t0`.`b` AS `b`,`test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b`,`test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b`,`test`.`t4`.`a` AS `a`,`test`.`t4`.`b` AS `b`,`test`.`t5`.`a` AS `a`,`test`.`t5`.`b` AS `b`,`test`.`t6`.`a` AS `a`,`test`.`t6`.`b` AS `b`,`test`.`t7`.`a` AS `a`,`test`.`t7`.`b` AS `b`,`test`.`t8`.`a` AS `a`,`test`.`t8`.`b` AS `b`,`test`.`t9`.`a` AS `a`,`test`.`t9`.`b` AS `b` from `test`.`t0` join `test`.`t1` left join (`test`.`t2` left join (`test`.`t3` join `test`.`t4`) on(((`test`.`t4`.`b` = `test`.`t2`.`b`) and (`test`.`t3`.`a` = 1))) join `test`.`t5` left join (`test`.`t6` join `test`.`t7` left join `test`.`t8` on(((`test`.`t8`.`b` = `test`.`t5`.`b`) and (`test`.`t6`.`b` < 10)))) on(((`test`.`t7`.`b` = `test`.`t5`.`b`) and (`test`.`t6`.`b` >= 2)))) on((((`test`.`t3`.`b` = 2) or isnull(`test`.`t3`.`c`)) and ((`test`.`t6`.`b` = 2) or isnull(`test`.`t6`.`c`)) and ((`test`.`t5`.`b` = `test`.`t0`.`b`) or isnull(`test`.`t3`.`c`) or isnull(`test`.`t6`.`c`) or isnull(`test`.`t8`.`c`)) and (`test`.`t1`.`a` <> 2))) join `test`.`t9` where ((`test`.`t9`.`a` = 1) and (`test`.`t1`.`b` = `test`.`t0`.`b`) and (`test`.`t0`.`a` = 1) and ((`test`.`t2`.`a` >= 4) or isnull(`test`.`t2`.`c`)) and ((`test`.`t3`.`a` < 5) or isnull(`test`.`t3`.`c`)) and ((`test`.`t3`.`b` = `test`.`t4`.`b`) or isnull(`test`.`t3`.`c`) or isnull(`test`.`t4`.`c`)) and ((`test`.`t5`.`a` >= 2) or isnull(`test`.`t5`.`c`)) and ((`test`.`t6`.`a` >= 4) or isnull(`test`.`t6`.`c`)) and ((`test`.`t7`.`a` <= 2) or isnull(`test`.`t7`.`c`)) and ((`test`.`t8`.`a` < 1) or isnull(`test`.`t8`.`c`)) and ((`test`.`t9`.`b` = `test`.`t8`.`b`) or isnull(`test`.`t8`.`c`))) Warnings: -Note 1003 select `test`.`t0`.`a` AS `a`,`test`.`t0`.`b` AS `b`,`test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b`,`test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t3`.`a` AS `a`,`test`.`t3`.`b` AS `b`,`test`.`t4`.`a` AS `a`,`test`.`t4`.`b` AS `b`,`test`.`t5`.`a` AS `a`,`test`.`t5`.`b` AS `b`,`test`.`t6`.`a` AS `a`,`test`.`t6`.`b` AS `b`,`test`.`t7`.`a` AS `a`,`test`.`t7`.`b` AS `b`,`test`.`t8`.`a` AS `a`,`test`.`t8`.`b` AS `b`,`test`.`t9`.`a` AS `a`,`test`.`t9`.`b` AS `b` from `test`.`t0` join `test`.`t1` left join (`test`.`t2` left join (`test`.`t3` join `test`.`t4`) on(((`test`.`t4`.`b` = `test`.`t2`.`b`) and (`test`.`t3`.`a` = 1))) join `test`.`t5` left join (`test`.`t6` join `test`.`t7` left join `test`.`t8` on(((`test`.`t8`.`b` = `test`.`t5`.`b`) and (`test`.`t6`.`b` < 10)))) on(((`test`.`t7`.`b` = `test`.`t5`.`b`) and (`test`.`t6`.`b` >= 2)))) on((((`test`.`t3`.`b` = 2) or isnull(`test`.`t3`.`c`)) and ((`test`.`t6`.`b` = 2) or isnull(`test`.`t6`.`c`)) and ((`test`.`t5`.`b` = `test`.`t0`.`b`) or isnull(`test`.`t3`.`c`) or isnull(`test`.`t6`.`c`) or isnull(`test`.`t8`.`c`)) and (`test`.`t1`.`a` <> 2))) join `test`.`t9` where ((`test`.`t9`.`a` = 1) and (`test`.`t1`.`b` = `test`.`t0`.`b`) and (`test`.`t0`.`a` = 1) and ((`test`.`t2`.`a` >= 4) or isnull(`test`.`t2`.`c`)) and ((`test`.`t3`.`a` < 5) or isnull(`test`.`t3`.`c`)) and ((`test`.`t4`.`b` = `test`.`t3`.`b`) or isnull(`test`.`t3`.`c`) or isnull(`test`.`t4`.`c`)) and ((`test`.`t5`.`a` >= 2) or isnull(`test`.`t5`.`c`)) and ((`test`.`t6`.`a` >= 4) or isnull(`test`.`t6`.`c`)) and ((`test`.`t7`.`a` <= 2) or isnull(`test`.`t7`.`c`)) and ((`test`.`t8`.`a` < 1) or isnull(`test`.`t8`.`c`)) and ((`test`.`t9`.`b` = `test`.`t8`.`b`) or isnull(`test`.`t8`.`c`))) SELECT t0.a,t0.b,t1.a,t1.b,t2.a,t2.b,t3.a,t3.b,t4.a,t4.b, t5.a,t5.b,t6.a,t6.b,t7.a,t7.b,t8.a,t8.b,t9.a,t9.b FROM t0,t1 @@ -1102,21 +1102,21 @@ t0.b=t1.b AND (t9.a=1); a b a b a b a b a b a b a b a b a b a b 1 2 2 2 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 1 1 -1 2 3 2 4 2 1 2 3 2 3 1 6 2 1 1 NULL NULL 1 1 -1 2 3 2 4 2 1 2 3 2 3 3 NULL NULL NULL NULL NULL NULL 1 1 -1 2 3 2 4 2 1 2 4 2 3 1 6 2 1 1 NULL NULL 1 1 -1 2 3 2 4 2 1 2 4 2 3 3 NULL NULL NULL NULL NULL NULL 1 1 -1 2 3 2 5 3 NULL NULL NULL NULL 3 1 6 2 1 1 NULL NULL 1 1 -1 2 3 2 5 3 NULL NULL NULL NULL 3 3 NULL NULL NULL NULL NULL NULL 1 1 1 2 2 2 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 1 2 -1 2 3 2 4 2 1 2 3 2 3 1 6 2 1 1 NULL NULL 1 2 1 2 3 2 4 2 1 2 3 2 2 2 6 2 2 2 0 2 1 2 +1 2 3 2 4 2 1 2 3 2 3 1 6 2 1 1 NULL NULL 1 1 +1 2 3 2 4 2 1 2 3 2 3 1 6 2 1 1 NULL NULL 1 2 +1 2 3 2 4 2 1 2 3 2 3 3 NULL NULL NULL NULL NULL NULL 1 1 1 2 3 2 4 2 1 2 3 2 3 3 NULL NULL NULL NULL NULL NULL 1 2 -1 2 3 2 4 2 1 2 4 2 3 1 6 2 1 1 NULL NULL 1 2 1 2 3 2 4 2 1 2 4 2 2 2 6 2 2 2 0 2 1 2 +1 2 3 2 4 2 1 2 4 2 3 1 6 2 1 1 NULL NULL 1 1 +1 2 3 2 4 2 1 2 4 2 3 1 6 2 1 1 NULL NULL 1 2 +1 2 3 2 4 2 1 2 4 2 3 3 NULL NULL NULL NULL NULL NULL 1 1 1 2 3 2 4 2 1 2 4 2 3 3 NULL NULL NULL NULL NULL NULL 1 2 -1 2 3 2 5 3 NULL NULL NULL NULL 3 1 6 2 1 1 NULL NULL 1 2 1 2 3 2 5 3 NULL NULL NULL NULL 2 2 6 2 2 2 0 2 1 2 +1 2 3 2 5 3 NULL NULL NULL NULL 3 1 6 2 1 1 NULL NULL 1 1 +1 2 3 2 5 3 NULL NULL NULL NULL 3 1 6 2 1 1 NULL NULL 1 2 +1 2 3 2 5 3 NULL NULL NULL NULL 3 3 NULL NULL NULL NULL NULL NULL 1 1 1 2 3 2 5 3 NULL NULL NULL NULL 3 3 NULL NULL NULL NULL NULL NULL 1 2 SELECT t2.a,t2.b FROM t2; @@ -1203,7 +1203,7 @@ EXPLAIN SELECT a, b, c FROM t1 LEFT JOIN id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 index NULL a 5 NULL 21 Using index 1 SIMPLE t3 index c c 5 NULL 6 Using index -1 SIMPLE t2 ref b b 5 test.t3.c 2 Using index +1 SIMPLE t2 ref b b 5 test.t3.c 1 Using index EXPLAIN SELECT a, b, c FROM t1 LEFT JOIN (t2, t3) ON b < 3 and b = c; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 index NULL a 5 NULL # Using index @@ -1484,8 +1484,8 @@ explain select * from t1 left join on (t1.a = t2.a); id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL NULL NULL NULL NULL 10 -1 SIMPLE t2 ref a a 5 test.t1.a 11 -1 SIMPLE t3 ref a a 5 test.t2.a 11 +1 SIMPLE t2 ref a a 5 test.t1.a 1 +1 SIMPLE t3 ref a a 5 test.t2.a 1 drop table t1, t2, t3; CREATE TABLE t1 (id int NOT NULL PRIMARY KEY, type varchar(10)); CREATE TABLE t2 (pid int NOT NULL PRIMARY KEY, type varchar(10)); === modified file 'mysql-test/suite/pbxt/r/key.result' --- a/mysql-test/suite/pbxt/r/key.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/key.result 2009-08-17 15:57:58 +0000 @@ -153,7 +153,7 @@ t1 0 PRIMARY 1 d A 0 NULL NULL BTREE t1 0 a 1 a A 0 NULL NULL BTREE t1 0 e 1 e A 0 NULL NULL BTREE t1 0 b 1 b A 0 NULL NULL YES BTREE -t1 1 c 1 c A NULL NULL NULL YES BTREE +t1 1 c 1 c A 0 NULL NULL YES BTREE drop table t1; CREATE TABLE t1 (c CHAR(10) NOT NULL,i INT NOT NULL AUTO_INCREMENT, UNIQUE (c,i)); === modified file 'mysql-test/suite/pbxt/r/key_cache.result' --- a/mysql-test/suite/pbxt/r/key_cache.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/key_cache.result 2009-08-17 15:57:58 +0000 @@ -122,7 +122,7 @@ i explain select count(*) from t1, t2 where t1.p = t2.i; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 index PRIMARY PRIMARY 4 NULL 2 Using index -1 SIMPLE t2 ref k1 k1 5 test.t1.p 2 Using where; Using index +1 SIMPLE t2 ref k1 k1 5 test.t1.p 1 Using where; Using index select count(*) from t1, t2 where t1.p = t2.i; count(*) 3 @@ -257,8 +257,6 @@ test.t2 assign_to_keycache note The stor drop table t1,t2,t3; set global keycache2.key_buffer_size=0; set global keycache3.key_buffer_size=100; -Warnings: -Warning 1292 Truncated incorrect key_buffer_size value: '100' set global keycache3.key_buffer_size=0; create table t1 (mytext text, FULLTEXT (mytext)) engine=myisam; insert t1 values ('aaabbb'); === modified file 'mysql-test/suite/pbxt/r/key_diff.result' --- a/mysql-test/suite/pbxt/r/key_diff.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/key_diff.result 2009-08-17 15:57:58 +0000 @@ -36,7 +36,7 @@ a a a a explain select t1.*,t2.* from t1,t1 as t2 where t1.A=t2.B; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL a NULL NULL NULL 5 -1 SIMPLE t2 ALL b NULL NULL NULL 5 Using where; Using join buffer +1 SIMPLE t2 ref b b 4 test.t1.a 1 Using where select t1.*,t2.* from t1,t1 as t2 where t1.A=t2.B order by binary t1.a,t2.a; a b a b A B a a === modified file 'mysql-test/suite/pbxt/r/lowercase_view.result' --- a/mysql-test/suite/pbxt/r/lowercase_view.result 2009-04-02 20:36:52 +0000 +++ b/mysql-test/suite/pbxt/r/lowercase_view.result 2009-08-31 11:07:44 +0000 @@ -119,7 +119,7 @@ create table t1Aa (col1 int); create view v1Aa as select col1 from t1Aa as AaA; show create view v1AA; View Create View character_set_client collation_connection -v1aa CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1aa` AS select `AaA`.`col1` AS `col1` from `t1aa` `AaA` latin1 latin1_swedish_ci +v1aa CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1aa` AS select `aaa`.`col1` AS `col1` from `t1aa` `aaa` latin1 latin1_swedish_ci drop view v1AA; select Aaa.col1 from t1Aa as AaA; col1 @@ -128,7 +128,7 @@ drop view v1AA; create view v1Aa as select AaA.col1 from t1Aa as AaA; show create view v1AA; View Create View character_set_client collation_connection -v1aa CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1aa` AS select `AaA`.`col1` AS `col1` from `t1aa` `AaA` latin1 latin1_swedish_ci +v1aa CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1aa` AS select `aaa`.`col1` AS `col1` from `t1aa` `aaa` latin1 latin1_swedish_ci drop view v1AA; drop table t1Aa; CREATE TABLE t1 (a int, b int); @@ -142,7 +142,7 @@ CREATE OR REPLACE VIEW v1 AS select X.a from t1 AS X group by X.b having (X.a = 1); SHOW CREATE VIEW v1; View Create View character_set_client collation_connection -v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select `X`.`a` AS `a` from `t1` `X` group by `X`.`b` having (`X`.`a` = 1) latin1 latin1_swedish_ci +v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select `x`.`a` AS `a` from `t1` `x` group by `x`.`b` having (`x`.`a` = 1) latin1 latin1_swedish_ci SELECT * FROM v1; a DROP VIEW v1; === modified file 'mysql-test/suite/pbxt/r/mysqlshow.result' --- a/mysql-test/suite/pbxt/r/mysqlshow.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/mysqlshow.result 2009-08-17 15:57:58 +0000 @@ -107,7 +107,21 @@ Database: information_schema | TRIGGERS | | USER_PRIVILEGES | | VIEWS | +| INNODB_BUFFER_POOL_PAGES | | PBXT_STATISTICS | +| INNODB_CMP | +| INNODB_RSEG | +| XTRADB_ENHANCEMENTS | +| INNODB_BUFFER_POOL_PAGES_INDEX | +| INNODB_INDEX_STATS | +| INNODB_TRX | +| INNODB_CMP_RESET | +| INNODB_LOCK_WAITS | +| INNODB_CMPMEM_RESET | +| INNODB_LOCKS | +| INNODB_CMPMEM | +| INNODB_TABLE_STATS | +| INNODB_BUFFER_POOL_PAGES_BLOB | +---------------------------------------+ Database: INFORMATION_SCHEMA +---------------------------------------+ @@ -141,7 +155,21 @@ Database: INFORMATION_SCHEMA | TRIGGERS | | USER_PRIVILEGES | | VIEWS | +| INNODB_BUFFER_POOL_PAGES | | PBXT_STATISTICS | +| INNODB_CMP | +| INNODB_RSEG | +| XTRADB_ENHANCEMENTS | +| INNODB_BUFFER_POOL_PAGES_INDEX | +| INNODB_INDEX_STATS | +| INNODB_TRX | +| INNODB_CMP_RESET | +| INNODB_LOCK_WAITS | +| INNODB_CMPMEM_RESET | +| INNODB_LOCKS | +| INNODB_CMPMEM | +| INNODB_TABLE_STATS | +| INNODB_BUFFER_POOL_PAGES_BLOB | +---------------------------------------+ Wildcard: inf_rmation_schema +--------------------+ === modified file 'mysql-test/suite/pbxt/r/null.result' --- a/mysql-test/suite/pbxt/r/null.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/null.result 2009-08-17 15:57:58 +0000 @@ -93,9 +93,11 @@ INSERT INTO t1 SET a = "", d= "2003-01-1 Warnings: Warning 1265 Data truncated for column 'd' at row 1 UPDATE t1 SET d=1/NULL; -ERROR 23000: Column 'd' cannot be null +Warnings: +Warning 1265 Data truncated for column 'd' at row 1 UPDATE t1 SET d=NULL; -ERROR 23000: Column 'd' cannot be null +Warnings: +Warning 1048 Column 'd' cannot be null INSERT INTO t1 (a) values (null); ERROR 23000: Column 'a' cannot be null INSERT INTO t1 (a) values (1/null); @@ -130,7 +132,7 @@ Warning 1048 Column 'd' cannot be null Warning 1048 Column 'd' cannot be null select * from t1; a b c d - 0 0000-00-00 00:00:00 2003 + 0 0000-00-00 00:00:00 0 0 0000-00-00 00:00:00 0 0 0000-00-00 00:00:00 0 0 0000-00-00 00:00:00 0 === modified file 'mysql-test/suite/pbxt/r/null_key.result' --- a/mysql-test/suite/pbxt/r/null_key.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/null_key.result 2009-08-17 15:57:58 +0000 @@ -407,8 +407,8 @@ EXPLAIN SELECT SQL_CALC_FOUND_ROWS * FRO LEFT JOIN t3 ON t2.b=t3.b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL NULL NULL NULL NULL 4 -1 SIMPLE t2 ref idx idx 5 test.t1.a 2 -1 SIMPLE t3 ref idx idx 5 test.t2.b 186 Using index +1 SIMPLE t2 ref idx idx 5 test.t1.a 1 +1 SIMPLE t3 ref idx idx 5 test.t2.b 1 Using index FLUSH STATUS ; SELECT SQL_CALC_FOUND_ROWS * FROM t1 LEFT JOIN t2 ON t1.a=t2.a LEFT JOIN t3 ON t2.b=t3.b; === modified file 'mysql-test/suite/pbxt/r/partition_pruning.result' --- a/mysql-test/suite/pbxt/r/partition_pruning.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/partition_pruning.result 2009-08-17 15:57:58 +0000 @@ -338,12 +338,12 @@ select * from t1 X, t1 Y where X.b = Y.b and (X.a=1 or X.a=2) and (Y.a=2 or Y.a=3); id select_type table partitions type possible_keys key key_len ref rows Extra 1 SIMPLE X p1,p2 ALL a,b NULL NULL NULL 2 Using where -1 SIMPLE Y p2,p3 ref a,b b 4 test.X.b 2 Using where +1 SIMPLE Y p2,p3 ref a,b b 4 test.X.b 1 Using where explain partitions select * from t1 X, t1 Y where X.a = Y.a and (X.a=1 or X.a=2); id select_type table partitions type possible_keys key key_len ref rows Extra 1 SIMPLE X p1,p2 ALL a NULL NULL NULL 4 Using where -1 SIMPLE Y p1,p2 ref a a 4 test.X.a 2 +1 SIMPLE Y p1,p2 ref a a 4 test.X.a 1 drop table t1; create table t1 (a int) partition by hash(a) partitions 20; insert into t1 values (1),(2),(3); === modified file 'mysql-test/suite/pbxt/r/pbxt_bugs.result' --- a/mysql-test/suite/pbxt/r/pbxt_bugs.result 2009-04-02 20:36:52 +0000 +++ b/mysql-test/suite/pbxt/r/pbxt_bugs.result 2009-08-17 15:57:58 +0000 @@ -1218,3 +1218,59 @@ c1 c2 0 opq 1 jkl DROP TABLE t1; +create table parent (id int primary key); +create table child (id int PRIMARY KEY, FOREIGN KEY (id) REFERENCES parent(id)); +insert into parent values (2), (3), (4); +insert into child values (3), (4); +delete ignore from parent; +ERROR 23000: Cannot delete or update a parent row: a foreign key constraint fails (Constraint: `FOREIGN_1`) +select * from parent; +id +2 +3 +4 +drop table child, parent; +create schema test378222; +use test378222; +create table t1 (id int primary key); +create table t2 (id int primary key); +alter table t1 add constraint foreign key (id) references t2 (id); +alter table t2 add constraint foreign key (id) references t1 (id); +drop schema test378222; +create schema test378222a; +create schema test378222b; +create table test378222a.t1 (id int primary key); +create table test378222b.t2 (id int primary key); +alter table test378222a.t1 add constraint foreign key (id) references test378222b.t2 (id); +alter table test378222b.t2 add constraint foreign key (id) references test378222a.t1 (id); +set foreign_key_checks = 1; +drop schema test378222a; +ERROR 23000: Cannot delete or update a parent row: a foreign key constraint fails +drop schema test378222b; +ERROR 23000: Cannot delete or update a parent row: a foreign key constraint fails +set foreign_key_checks = 0; +drop schema test378222a; +drop schema test378222b; +set foreign_key_checks = 1; +use test; +CREATE TABLE t1(c1 TINYINT AUTO_INCREMENT NULL KEY ) AUTO_INCREMENT=10; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c1` tinyint(4) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`c1`) +) ENGINE=PBXT AUTO_INCREMENT=10 DEFAULT CHARSET=latin1 +INSERT INTO t1 VALUES(null); +INSERT INTO t1 VALUES(null); +INSERT INTO t1 VALUES(null); +SELECT * FROM t1; +c1 +10 +11 +12 +TRUNCATE TABLE t1; +INSERT INTO t1 VALUES(null); +SELECT * FROM t1; +c1 +1 +DROP TABLE t1; === modified file 'mysql-test/suite/pbxt/r/pbxt_ref_int.result' --- a/mysql-test/suite/pbxt/r/pbxt_ref_int.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/pbxt_ref_int.result 2009-08-17 15:57:58 +0000 @@ -166,7 +166,7 @@ child CREATE TABLE `child` ( `parent_id` int(11) DEFAULT NULL, KEY `par_ind` (`parent_id`), KEY `child_ind` (`id`), - CONSTRAINT `FOREIGN_1` FOREIGN KEY (`parent_id`) REFERENCES `parent` (`id`) + CONSTRAINT `FOREIGN_1` FOREIGN KEY (`parent_id`) REFERENCES `test`.`parent` (`id`) ) ENGINE=PBXT DEFAULT CHARSET=latin1 drop index child_ind on child; show create table child; @@ -175,7 +175,7 @@ child CREATE TABLE `child` ( `id` int(11) DEFAULT NULL, `parent_id` int(11) DEFAULT NULL, KEY `par_ind` (`parent_id`), - CONSTRAINT `FOREIGN_1` FOREIGN KEY (`parent_id`) REFERENCES `parent` (`id`) + CONSTRAINT `FOREIGN_1` FOREIGN KEY (`parent_id`) REFERENCES `test`.`parent` (`id`) ) ENGINE=PBXT DEFAULT CHARSET=latin1 alter table parent add column c1 varchar(40); insert child values(2000, 2); @@ -243,7 +243,7 @@ child CREATE TABLE `child` ( `id` int(11) DEFAULT NULL, `parent_id` int(11) DEFAULT NULL, KEY `par_ind` (`parent_id`), - CONSTRAINT `FOREIGN_1` FOREIGN KEY (`parent_id`) REFERENCES `parent` (`id`) + CONSTRAINT `FOREIGN_1` FOREIGN KEY (`parent_id`) REFERENCES `test`.`parent` (`id`) ) ENGINE=PBXT DEFAULT CHARSET=latin1 alter table child add column c1 varchar(40); insert child values(400, 1, "asd"); @@ -284,7 +284,7 @@ child CREATE TABLE `child` ( `id` int(11) DEFAULT NULL, `parent_id` int(11) DEFAULT NULL, KEY `par_ind` (`parent_id`), - CONSTRAINT `FOREIGN_1` FOREIGN KEY (`parent_id`) REFERENCES `parent` (`id`) ON DELETE CASCADE + CONSTRAINT `FOREIGN_1` FOREIGN KEY (`parent_id`) REFERENCES `test`.`parent` (`id`) ON DELETE CASCADE ) ENGINE=PBXT DEFAULT CHARSET=latin1 insert parent values(1); insert child values(100, 1); === modified file 'mysql-test/suite/pbxt/r/preload.result' --- a/mysql-test/suite/pbxt/r/preload.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/preload.result 2009-08-17 15:57:58 +0000 @@ -144,7 +144,7 @@ Key_reads 0 load index into cache t3, t2 key (primary,b) ; Table Op Msg_type Msg_text test.t3 preload_keys Error Table 'test.t3' doesn't exist -test.t3 preload_keys error Corrupt +test.t3 preload_keys status Operation failed test.t2 preload_keys note The storage engine for the table doesn't support preload_keys show status like "key_read%"; Variable_name Value @@ -159,7 +159,7 @@ Key_reads 0 load index into cache t3 key (b), t2 key (c) ; Table Op Msg_type Msg_text test.t3 preload_keys Error Table 'test.t3' doesn't exist -test.t3 preload_keys error Corrupt +test.t3 preload_keys status Operation failed test.t2 preload_keys note The storage engine for the table doesn't support preload_keys show status like "key_read%"; Variable_name Value === modified file 'mysql-test/suite/pbxt/r/ps_1general.result' --- a/mysql-test/suite/pbxt/r/ps_1general.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/ps_1general.result 2009-08-31 11:07:44 +0000 @@ -289,11 +289,11 @@ prepare stmt4 from ' show index from t2 execute stmt4; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment t2 0 PRIMARY 1 a A 0 NULL NULL BTREE -t2 1 t2_idx 1 b A NULL NULL NULL YES BTREE +t2 1 t2_idx 1 b A 0 NULL NULL YES BTREE prepare stmt4 from ' show table status from test like ''t2%'' '; execute stmt4; Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment -t2 PBXT 10 Fixed 0 29 1 # 4096 0 NULL # # # latin1_swedish_ci NULL +t2 PBXT 10 Fixed 0 29 1024 # 4096 0 NULL # # # latin1_swedish_ci NULL prepare stmt4 from ' show table status from test like ''t9%'' '; execute stmt4; Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment @@ -447,7 +447,7 @@ def type 253 10 3 Y 0 31 8 def possible_keys 253 4096 0 Y 0 31 8 def key 253 64 0 Y 0 31 8 def key_len 253 4096 0 Y 0 31 8 -def ref 253 1024 0 Y 0 31 8 +def ref 253 2048 0 Y 0 31 8 def rows 8 10 1 Y 32928 0 63 def Extra 253 255 14 N 1 31 8 id select_type table type possible_keys key key_len ref rows Extra @@ -463,7 +463,7 @@ def type 253 10 5 Y 0 31 8 def possible_keys 253 4096 7 Y 0 31 8 def key 253 64 7 Y 0 31 8 def key_len 253 4096 1 Y 0 31 8 -def ref 253 1024 0 Y 0 31 8 +def ref 253 2048 0 Y 0 31 8 def rows 8 10 1 Y 32928 0 63 def Extra 253 255 27 N 1 31 8 id select_type table type possible_keys key key_len ref rows Extra === modified file 'mysql-test/suite/pbxt/r/range.result' --- a/mysql-test/suite/pbxt/r/range.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/range.result 2009-08-17 15:57:58 +0000 @@ -423,19 +423,19 @@ test.t2 analyze status OK explain select * from t1, t2 where t1.uid=t2.uid AND t1.uid > 0; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 range uid_index uid_index 4 NULL 1 Using where -1 SIMPLE t2 ref uid_index uid_index 4 test.t1.uid 12 +1 SIMPLE t2 ref uid_index uid_index 4 test.t1.uid 1 explain select * from t1, t2 where t1.uid=t2.uid AND t2.uid > 0; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 range uid_index uid_index 4 NULL 1 Using where -1 SIMPLE t2 ref uid_index uid_index 4 test.t1.uid 12 +1 SIMPLE t2 ref uid_index uid_index 4 test.t1.uid 1 explain select * from t1, t2 where t1.uid=t2.uid AND t1.uid != 0; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 range uid_index uid_index 4 NULL 2 Using where -1 SIMPLE t2 ref uid_index uid_index 4 test.t1.uid 12 +1 SIMPLE t2 ref uid_index uid_index 4 test.t1.uid 1 explain select * from t1, t2 where t1.uid=t2.uid AND t2.uid != 0; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 range uid_index uid_index 4 NULL 2 Using where -1 SIMPLE t2 ref uid_index uid_index 4 test.t1.uid 12 +1 SIMPLE t2 ref uid_index uid_index 4 test.t1.uid 1 select * from t1, t2 where t1.uid=t2.uid AND t1.uid > 0; id name uid id name uid 1001 A 1 1001 A 1 === modified file 'mysql-test/suite/pbxt/r/schema.result' --- a/mysql-test/suite/pbxt/r/schema.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/schema.result 2009-08-17 15:57:58 +0000 @@ -3,11 +3,13 @@ create schema foo; show create schema foo; Database Create Database foo CREATE DATABASE `foo` /*!40100 DEFAULT CHARACTER SET latin1 */ +create table t1 (id int) engine=pbxt; show schemas; Database information_schema foo mtr mysql +pbxt test drop schema foo; === modified file 'mysql-test/suite/pbxt/r/select.result' --- a/mysql-test/suite/pbxt/r/select.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/select.result 2009-08-17 15:57:58 +0000 @@ -604,15 +604,15 @@ id select_type table type possible_keys explain select * from t3 as t1,t3 where t1.period=t3.period order by t3.period; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL period NULL NULL NULL 41810 Using temporary; Using filesort -1 SIMPLE t3 ref period period 4 test.t1.period 18 +1 SIMPLE t3 ref period period 4 test.t1.period 1 explain select * from t3 as t1,t3 where t1.period=t3.period order by t3.period limit 10; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t3 index period period 4 NULL 1 -1 SIMPLE t1 ref period period 4 test.t3.period 18 +1 SIMPLE t3 index period period 4 NULL 10 +1 SIMPLE t1 ref period period 4 test.t3.period 1 explain select * from t3 as t1,t3 where t1.period=t3.period order by t1.period limit 10; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 index period period 4 NULL 1 -1 SIMPLE t3 ref period period 4 test.t1.period 18 +1 SIMPLE t1 index period period 4 NULL 10 +1 SIMPLE t3 ref period period 4 test.t1.period 1 select period from t1; period 9410 @@ -2095,7 +2095,7 @@ show keys from t2; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment t2 0 PRIMARY 1 auto A 1199 NULL NULL BTREE t2 0 fld1 1 fld1 A 1199 NULL NULL BTREE -t2 1 fld3 1 fld3 A NULL NULL NULL BTREE +t2 1 fld3 1 fld3 A 1199 NULL NULL BTREE drop table t4, t3, t2, t1; DO 1; DO benchmark(100,1+1),1,1; @@ -2369,7 +2369,7 @@ insert into t2 values (1,3), (2,3), (3,4 explain select * from t1 left join t2 on a=c where d in (4); id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t2 ref c,d d 5 const 1 Using where -1 SIMPLE t1 ref a a 5 test.t2.c 2 Using where +1 SIMPLE t1 ref a a 5 test.t2.c 1 Using where select * from t1 left join t2 on a=c where d in (4); a b c d 3 2 3 4 @@ -2377,7 +2377,7 @@ a b c d explain select * from t1 left join t2 on a=c where d = 4; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t2 ref c,d d 5 const 1 Using where -1 SIMPLE t1 ref a a 5 test.t2.c 2 Using where +1 SIMPLE t1 ref a a 5 test.t2.c 1 Using where select * from t1 left join t2 on a=c where d = 4; a b c d 3 2 3 4 @@ -2403,11 +2403,11 @@ INSERT INTO t2 VALUES ('one'),('two'),(' EXPLAIN SELECT * FROM t1 LEFT JOIN t2 USE INDEX (a) ON t1.a=t2.a; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL NULL NULL NULL NULL 5 -1 SIMPLE t2 ref a a 23 test.t1.a 2 +1 SIMPLE t2 ref a a 23 test.t1.a 1 EXPLAIN SELECT * FROM t1 LEFT JOIN t2 FORCE INDEX (a) ON t1.a=t2.a; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL NULL NULL NULL NULL 5 -1 SIMPLE t2 ref a a 23 test.t1.a 2 +1 SIMPLE t2 ref a a 23 test.t1.a 1 DROP TABLE t1, t2; CREATE TABLE t1 ( city char(30) ); INSERT INTO t1 VALUES ('London'); @@ -2792,26 +2792,26 @@ id select_type table type possible_keys 1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Select tables optimized away select max(key1) from t1 where key1 <= 0.6158; max(key1) -0.61580002307892 +0.615800023078918 select max(key2) from t2 where key2 <= 1.6158; max(key2) -1.6158000230789 +1.61580002307892 select min(key1) from t1 where key1 >= 0.3762; min(key1) -0.37619999051094 +0.376199990510941 select min(key2) from t2 where key2 >= 1.3762; min(key2) -1.3761999607086 +1.37619996070862 select max(key1), min(key2) from t1, t2 where key1 <= 0.6158 and key2 >= 1.3762; max(key1) min(key2) -0.61580002307892 1.3761999607086 +0.615800023078918 1.37619996070862 select max(key1) from t1 where key1 <= 0.6158 and rand() + 0.5 >= 0.5; max(key1) -0.61580002307892 +0.615800023078918 select min(key1) from t1 where key1 >= 0.3762 and rand() + 0.5 >= 0.5; min(key1) -0.37619999051094 +0.376199990510941 DROP TABLE t1,t2; CREATE TABLE t1 (i BIGINT UNSIGNED NOT NULL); INSERT INTO t1 VALUES (10); @@ -3454,7 +3454,7 @@ explain select * from t2 A, t2 B where A and B.a=5 and B.b=A.e and (B.b =1 or B.b = 3 or B.b=5); id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE A range PRIMARY PRIMARY 12 NULL 1 Using where -1 SIMPLE B ref PRIMARY PRIMARY 8 const,test.A.e 11 +1 SIMPLE B ref PRIMARY PRIMARY 8 const,test.A.e 1 drop table t1, t2; CREATE TABLE t1 (a int PRIMARY KEY, b int, INDEX(b)); INSERT INTO t1 VALUES (1, 3), (9,4), (7,5), (4,5), (6,2), @@ -3468,12 +3468,12 @@ EXPLAIN SELECT a, c, d, f FROM t1,t2 WHERE a=c AND b BETWEEN 4 AND 6; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 range PRIMARY,b b 5 NULL 1 Using where -1 SIMPLE t2 ref c c 5 test.t1.a 2 Using where +1 SIMPLE t2 ref c c 5 test.t1.a 1 Using where EXPLAIN SELECT a, c, d, f FROM t1,t2 WHERE a=c AND b BETWEEN 4 AND 6 AND a > 0; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 range PRIMARY,b PRIMARY 4 NULL 1 Using where -1 SIMPLE t2 ref c c 5 test.t1.a 2 Using where +1 SIMPLE t2 ref c c 5 test.t1.a 1 Using where DROP TABLE t1, t2; create table t1 ( a int unsigned not null auto_increment primary key, === modified file 'mysql-test/suite/pbxt/r/select_safe.result' --- a/mysql-test/suite/pbxt/r/select_safe.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/select_safe.result 2009-08-17 15:57:58 +0000 @@ -70,12 +70,12 @@ insert into t1 values (null,"a"),(null," explain select STRAIGHT_JOIN * from t1,t1 as t2 where t1.b=t2.b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL b NULL NULL NULL 21 -1 SIMPLE t2 ref b b 21 test.t1.b 2 Using where +1 SIMPLE t2 ref b b 21 test.t1.b 1 Using where set MAX_SEEKS_FOR_KEY=1; explain select STRAIGHT_JOIN * from t1,t1 as t2 where t1.b=t2.b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL b NULL NULL NULL 21 -1 SIMPLE t2 ref b b 21 test.t1.b 2 Using where +1 SIMPLE t2 ref b b 21 test.t1.b 1 Using where SET MAX_SEEKS_FOR_KEY=DEFAULT; drop table t1; create table t1 (a int); === modified file 'mysql-test/suite/pbxt/r/subselect.result' --- a/mysql-test/suite/pbxt/r/subselect.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/subselect.result 2009-08-17 15:57:58 +0000 @@ -1333,7 +1333,7 @@ a explain extended select * from t2 where t2.a in (select a from t1); id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t2 index NULL a 5 NULL 4 100.00 Using where; Using index -2 DEPENDENT SUBQUERY t1 index_subquery a a 5 func 1001 100.00 Using index; Using where +2 DEPENDENT SUBQUERY t1 index_subquery a a 5 func 1 100.00 Using index; Using where Warnings: Note 1003 select `test`.`t2`.`a` AS `a` from `test`.`t2` where <in_optimizer>(`test`.`t2`.`a`,<exists>(<index_lookup>(<cache>(`test`.`t2`.`a`) in t1 on a where (<cache>(`test`.`t2`.`a`) = `test`.`t1`.`a`)))) select * from t2 where t2.a in (select a from t1 where t1.b <> 30); @@ -1343,7 +1343,7 @@ a explain extended select * from t2 where t2.a in (select a from t1 where t1.b <> 30); id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t2 index NULL a 5 NULL 4 100.00 Using where; Using index -2 DEPENDENT SUBQUERY t1 index_subquery a a 5 func 1001 100.00 Using index; Using where +2 DEPENDENT SUBQUERY t1 index_subquery a a 5 func 1 100.00 Using index; Using where Warnings: Note 1003 select `test`.`t2`.`a` AS `a` from `test`.`t2` where <in_optimizer>(`test`.`t2`.`a`,<exists>(<index_lookup>(<cache>(`test`.`t2`.`a`) in t1 on a where ((`test`.`t1`.`b` <> 30) and (<cache>(`test`.`t2`.`a`) = `test`.`t1`.`a`))))) select * from t2 where t2.a in (select t1.a from t1,t3 where t1.b=t3.a); @@ -1353,8 +1353,8 @@ a explain extended select * from t2 where t2.a in (select t1.a from t1,t3 where t1.b=t3.a); id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t2 index NULL a 5 NULL 4 100.00 Using where; Using index -2 DEPENDENT SUBQUERY t1 ref a a 5 func 1001 100.00 Using where; Using index -2 DEPENDENT SUBQUERY t3 index a a 5 NULL 3 100.00 Using where; Using index; Using join buffer +2 DEPENDENT SUBQUERY t1 ref a a 5 func 1 100.00 Using where; Using index +2 DEPENDENT SUBQUERY t3 ref a a 5 test.t1.b 1 100.00 Using where; Using index Warnings: Note 1003 select `test`.`t2`.`a` AS `a` from `test`.`t2` where <in_optimizer>(`test`.`t2`.`a`,<exists>(select 1 AS `Not_used` from `test`.`t1` join `test`.`t3` where ((`test`.`t3`.`a` = `test`.`t1`.`b`) and (<cache>(`test`.`t2`.`a`) = `test`.`t1`.`a`)))) insert into t1 values (3,31); @@ -1370,7 +1370,7 @@ a explain extended select * from t2 where t2.a in (select a from t1 where t1.b <> 30); id select_type table type possible_keys key key_len ref rows filtered Extra 1 PRIMARY t2 index NULL a 5 NULL 4 100.00 Using where; Using index -2 DEPENDENT SUBQUERY t1 index_subquery a a 5 func 1001 100.00 Using index; Using where +2 DEPENDENT SUBQUERY t1 index_subquery a a 5 func 1 100.00 Using index; Using where Warnings: Note 1003 select `test`.`t2`.`a` AS `a` from `test`.`t2` where <in_optimizer>(`test`.`t2`.`a`,<exists>(<index_lookup>(<cache>(`test`.`t2`.`a`) in t1 on a where ((`test`.`t1`.`b` <> 30) and (<cache>(`test`.`t2`.`a`) = `test`.`t1`.`a`))))) drop table t1, t2, t3; @@ -3546,7 +3546,7 @@ ORDER BY t1.t DESC LIMIT 1); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 1 1 PRIMARY t1 index NULL PRIMARY 16 NULL 11 Using where; Using index; Using join buffer -2 DEPENDENT SUBQUERY t1 ref PRIMARY PRIMARY 8 test.t2.i1,const 2 Using where; Using index; Using filesort +2 DEPENDENT SUBQUERY t1 ref PRIMARY PRIMARY 8 test.t2.i1,const 1 Using where; Using index; Using filesort SELECT * FROM t1,t2 WHERE t1.t = (SELECT t1.t FROM t1 WHERE t1.t < t2.t AND t1.i2=1 AND t2.i1=t1.i1 @@ -4214,7 +4214,7 @@ CREATE INDEX I2 ON t1 (b); EXPLAIN SELECT a,b FROM t1 WHERE b IN (SELECT a FROM t1); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 2 Using where -2 DEPENDENT SUBQUERY t1 index_subquery I1 I1 2 func 2 Using index; Using where +2 DEPENDENT SUBQUERY t1 index_subquery I1 I1 2 func 1 Using index; Using where SELECT a,b FROM t1 WHERE b IN (SELECT a FROM t1); a b CREATE TABLE t2 (a VARCHAR(1), b VARCHAR(10)); @@ -4224,14 +4224,14 @@ CREATE INDEX I2 ON t2 (b); EXPLAIN SELECT a,b FROM t2 WHERE b IN (SELECT a FROM t2); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t2 ALL NULL NULL NULL NULL 2 Using where -2 DEPENDENT SUBQUERY t2 index_subquery I1 I1 4 func 2 Using index; Using where +2 DEPENDENT SUBQUERY t2 index_subquery I1 I1 4 func 1 Using index; Using where SELECT a,b FROM t2 WHERE b IN (SELECT a FROM t2); a b EXPLAIN SELECT a,b FROM t1 WHERE b IN (SELECT a FROM t1 WHERE LENGTH(a)<500); id select_type table type possible_keys key key_len ref rows Extra 1 PRIMARY t1 ALL NULL NULL NULL NULL 2 Using where -2 DEPENDENT SUBQUERY t1 index_subquery I1 I1 2 func 2 Using index; Using where +2 DEPENDENT SUBQUERY t1 index_subquery I1 I1 2 func 1 Using index; Using where SELECT a,b FROM t1 WHERE b IN (SELECT a FROM t1 WHERE LENGTH(a)<500); a b DROP TABLE t1,t2; === modified file 'mysql-test/suite/pbxt/r/type_enum.result' --- a/mysql-test/suite/pbxt/r/type_enum.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/type_enum.result 2009-08-17 15:57:58 +0000 @@ -1776,8 +1776,14 @@ t1 CREATE TABLE `t1` ( `russian_deviant` enum('E','F','E�F','F,E') NOT NULL DEFAULT 'E' ) ENGINE=PBXT DEFAULT CHARSET=latin1 drop table t1; +select @@SESSION.sql_mode; +@@SESSION.sql_mode + +select @@GLOBAL.sql_mode; +@@GLOBAL.sql_mode + create table t1(exhausting_charset enum('ABCDEFGHIJKLMNOPQRSTUVWXYZ',' !"','#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~','xx\','yy\�','zz��')); -ERROR HY000: Can't create table 'test.t1' (errno: -1) +drop table t1; End of 5.1 tests === modified file 'mysql-test/suite/pbxt/r/type_ranges.result' --- a/mysql-test/suite/pbxt/r/type_ranges.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/type_ranges.result 2009-08-17 15:57:58 +0000 @@ -70,19 +70,19 @@ flags set('one','two','tree') latin1_swe show keys from t1; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment t1 0 PRIMARY 1 auto A 0 NULL NULL BTREE -t1 1 utiny 1 utiny A NULL NULL NULL BTREE -t1 1 tiny 1 tiny A NULL NULL NULL BTREE -t1 1 short 1 short A NULL NULL NULL BTREE -t1 1 any_name 1 medium A NULL NULL NULL BTREE -t1 1 longlong 1 longlong A NULL NULL NULL BTREE -t1 1 real_float 1 real_float A NULL NULL NULL BTREE -t1 1 ushort 1 ushort A NULL NULL NULL BTREE -t1 1 umedium 1 umedium A NULL NULL NULL BTREE -t1 1 ulong 1 ulong A NULL NULL NULL BTREE -t1 1 ulonglong 1 ulonglong A NULL NULL NULL BTREE -t1 1 ulonglong 2 ulong A NULL NULL NULL BTREE -t1 1 options 1 options A NULL NULL NULL BTREE -t1 1 options 2 flags A NULL NULL NULL BTREE +t1 1 utiny 1 utiny A 0 NULL NULL BTREE +t1 1 tiny 1 tiny A 0 NULL NULL BTREE +t1 1 short 1 short A 0 NULL NULL BTREE +t1 1 any_name 1 medium A 0 NULL NULL BTREE +t1 1 longlong 1 longlong A 0 NULL NULL BTREE +t1 1 real_float 1 real_float A 0 NULL NULL BTREE +t1 1 ushort 1 ushort A 0 NULL NULL BTREE +t1 1 umedium 1 umedium A 0 NULL NULL BTREE +t1 1 ulong 1 ulong A 0 NULL NULL BTREE +t1 1 ulonglong 1 ulonglong A 0 NULL NULL BTREE +t1 1 ulonglong 2 ulong A 0 NULL NULL BTREE +t1 1 options 1 options A 0 NULL NULL BTREE +t1 1 options 2 flags A 0 NULL NULL BTREE CREATE UNIQUE INDEX test on t1 ( auto ) ; CREATE INDEX test2 on t1 ( ulonglong,ulong) ; CREATE INDEX test3 on t1 ( medium ) ; === modified file 'mysql-test/suite/pbxt/r/type_timestamp.result' --- a/mysql-test/suite/pbxt/r/type_timestamp.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/type_timestamp.result 2009-08-17 15:57:58 +0000 @@ -101,13 +101,13 @@ create table t1 (t2 timestamp(2), t4 tim t8 timestamp(8), t10 timestamp(10), t12 timestamp(12), t14 timestamp(14)); Warnings: -Warning 1287 The syntax 'TIMESTAMP(2)' is deprecated and will be removed in MySQL 5.2. Please use 'TIMESTAMP' instead -Warning 1287 The syntax 'TIMESTAMP(4)' is deprecated and will be removed in MySQL 5.2. Please use 'TIMESTAMP' instead -Warning 1287 The syntax 'TIMESTAMP(6)' is deprecated and will be removed in MySQL 5.2. Please use 'TIMESTAMP' instead -Warning 1287 The syntax 'TIMESTAMP(8)' is deprecated and will be removed in MySQL 5.2. Please use 'TIMESTAMP' instead -Warning 1287 The syntax 'TIMESTAMP(10)' is deprecated and will be removed in MySQL 5.2. Please use 'TIMESTAMP' instead -Warning 1287 The syntax 'TIMESTAMP(12)' is deprecated and will be removed in MySQL 5.2. Please use 'TIMESTAMP' instead -Warning 1287 The syntax 'TIMESTAMP(14)' is deprecated and will be removed in MySQL 5.2. Please use 'TIMESTAMP' instead +Warning 1287 The syntax 'TIMESTAMP(2)' is deprecated and will be removed in MySQL 6.0. Please use 'TIMESTAMP' instead +Warning 1287 The syntax 'TIMESTAMP(4)' is deprecated and will be removed in MySQL 6.0. Please use 'TIMESTAMP' instead +Warning 1287 The syntax 'TIMESTAMP(6)' is deprecated and will be removed in MySQL 6.0. Please use 'TIMESTAMP' instead +Warning 1287 The syntax 'TIMESTAMP(8)' is deprecated and will be removed in MySQL 6.0. Please use 'TIMESTAMP' instead +Warning 1287 The syntax 'TIMESTAMP(10)' is deprecated and will be removed in MySQL 6.0. Please use 'TIMESTAMP' instead +Warning 1287 The syntax 'TIMESTAMP(12)' is deprecated and will be removed in MySQL 6.0. Please use 'TIMESTAMP' instead +Warning 1287 The syntax 'TIMESTAMP(14)' is deprecated and will be removed in MySQL 6.0. Please use 'TIMESTAMP' instead insert t1 values (0,0,0,0,0,0,0), ("1997-12-31 23:47:59", "1997-12-31 23:47:59", "1997-12-31 23:47:59", "1997-12-31 23:47:59", "1997-12-31 23:47:59", "1997-12-31 23:47:59", === modified file 'mysql-test/suite/pbxt/r/union.result' --- a/mysql-test/suite/pbxt/r/union.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/union.result 2009-08-17 15:57:58 +0000 @@ -1301,12 +1301,14 @@ t3 CREATE TABLE `t3` ( `left(a,100000000)` longtext ) ENGINE=PBXT DEFAULT CHARSET=latin1 drop tables t1,t2,t3; +SELECT @tmp_max:= @@global.max_allowed_packet; +@tmp_max:= @@global.max_allowed_packet +1048576 +SET @@global.max_allowed_packet=25000000; CREATE TABLE t1 (a mediumtext); CREATE TABLE t2 (b varchar(20)); INSERT INTO t1 VALUES ('a'); CREATE TABLE t3 SELECT REPEAT(a,20000000) AS a FROM t1 UNION SELECT b FROM t2; -Warnings: -Warning 1301 Result of repeat() was larger than max_allowed_packet (1048576) - truncated SHOW CREATE TABLE t3; Table Create Table t3 CREATE TABLE `t3` ( @@ -1340,6 +1342,7 @@ t3 CREATE TABLE `t3` ( `a` varbinary(510) DEFAULT NULL ) ENGINE=PBXT DEFAULT CHARSET=latin1 DROP TABLES t1,t2,t3; +SET @@global.max_allowed_packet:= @tmp_max; create table t1 ( id int not null auto_increment, primary key (id), col1 int); insert into t1 (col1) values (2),(3),(4),(5),(6); select 99 union all select id from t1 order by 1; === modified file 'mysql-test/suite/pbxt/r/view_grant.result' --- a/mysql-test/suite/pbxt/r/view_grant.result 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/r/view_grant.result 2009-08-17 15:57:58 +0000 @@ -28,7 +28,7 @@ create view v2 as select * from mysqltes ERROR 42000: ANY command denied to user 'mysqltest_1'@'localhost' for table 't2' show create view v1; View Create View character_set_client collation_connection -v1 CREATE ALGORITHM=UNDEFINED DEFINER=`mysqltest_1`@`localhost` SQL SECURITY DEFINER VIEW `test`.`v1` AS select `mysqltest`.`t1`.`a` AS `a`,`mysqltest`.`t1`.`b` AS `b` from `mysqltest`.`t1` latin1 latin1_swedish_ci +v1 CREATE ALGORITHM=UNDEFINED DEFINER=`mysqltest_1`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select `mysqltest`.`t1`.`a` AS `a`,`mysqltest`.`t1`.`b` AS `b` from `mysqltest`.`t1` latin1 latin1_swedish_ci grant create view,drop,select on test.* to mysqltest_1@localhost; use test; alter view v1 as select * from mysqltest.t1; @@ -309,7 +309,7 @@ grant create view,select on test.* to my create view v1 as select * from mysqltest.t1; show create view v1; View Create View character_set_client collation_connection -v1 CREATE ALGORITHM=UNDEFINED DEFINER=`mysqltest_1`@`localhost` SQL SECURITY DEFINER VIEW `test`.`v1` AS select `mysqltest`.`t1`.`a` AS `a`,`mysqltest`.`t1`.`b` AS `b` from `mysqltest`.`t1` latin1 latin1_swedish_ci +v1 CREATE ALGORITHM=UNDEFINED DEFINER=`mysqltest_1`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select `mysqltest`.`t1`.`a` AS `a`,`mysqltest`.`t1`.`b` AS `b` from `mysqltest`.`t1` latin1 latin1_swedish_ci revoke select on mysqltest.t1 from mysqltest_1@localhost; select * from v1; ERROR HY000: View 'test.v1' references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them === modified file 'mysql-test/suite/pbxt/t/auto_increment.test' --- a/mysql-test/suite/pbxt/t/auto_increment.test 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/t/auto_increment.test 2009-08-17 15:57:58 +0000 @@ -150,7 +150,6 @@ delete from t1 where a=0; update t1 set a=0 where b=5; select * from t1 order by b; delete from t1 where a=0; ---error 1048 update t1 set a=NULL where b=6; update t1 set a=300 where b=7; SET SQL_MODE=''; @@ -166,7 +165,6 @@ delete from t1 where a=0; update t1 set a=0 where b=12; select * from t1 order by b; delete from t1 where a=0; ---error 1048 update t1 set a=NULL where b=13; update t1 set a=500 where b=14; select * from t1 order by b; === modified file 'mysql-test/suite/pbxt/t/delete.test' --- a/mysql-test/suite/pbxt/t/delete.test 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/t/delete.test 2009-08-17 15:57:58 +0000 @@ -120,13 +120,16 @@ select * from t2; delete t11.*, t12.* from t11,t12 where t11.a = t12.a and t11.b <> (select b from t2 where t11.a < t2.a); select * from t11; select * from t12; +--error 1242 delete ignore t11.*, t12.* from t11,t12 where t11.a = t12.a and t11.b <> (select b from t2 where t11.a < t2.a); select * from t11; select * from t12; +--error 1062 insert into t11 values (2, 12); -- error 1242 delete from t11 where t11.b <> (select b from t2 where t11.a < t2.a); select * from t11; +--error 1242 delete ignore from t11 where t11.b <> (select b from t2 where t11.a < t2.a); select * from t11; drop table t11, t12, t2; === modified file 'mysql-test/suite/pbxt/t/join_nested.test' --- a/mysql-test/suite/pbxt/t/join_nested.test 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/t/join_nested.test 2009-08-17 15:57:58 +0000 @@ -546,6 +546,7 @@ SELECT t0.a,t0.b,t1.a,t1.b,t2.a,t2.b,t3. CREATE INDEX idx_b ON t8(b); +--sorted_result EXPLAIN EXTENDED SELECT t0.a,t0.b,t1.a,t1.b,t2.a,t2.b,t3.a,t3.b,t4.a,t4.b, t5.a,t5.b,t6.a,t6.b,t7.a,t7.b,t8.a,t8.b,t9.a,t9.b @@ -585,6 +586,7 @@ SELECT t0.a,t0.b,t1.a,t1.b,t2.a,t2.b,t3. CREATE INDEX idx_b ON t1(b); CREATE INDEX idx_a ON t0(a); +--sorted_result EXPLAIN EXTENDED SELECT t0.a,t0.b,t1.a,t1.b,t2.a,t2.b,t3.a,t3.b,t4.a,t4.b, t5.a,t5.b,t6.a,t6.b,t7.a,t7.b,t8.a,t8.b,t9.a,t9.b @@ -621,6 +623,7 @@ SELECT t0.a,t0.b,t1.a,t1.b,t2.a,t2.b,t3. (t8.b=t9.b OR t8.c IS NULL) AND (t9.a=1); +--sorted_result SELECT t0.a,t0.b,t1.a,t1.b,t2.a,t2.b,t3.a,t3.b,t4.a,t4.b, t5.a,t5.b,t6.a,t6.b,t7.a,t7.b,t8.a,t8.b,t9.a,t9.b FROM t0,t1 === added file 'mysql-test/suite/pbxt/t/lowercase_table_grant-master.opt' --- a/mysql-test/suite/pbxt/t/lowercase_table_grant-master.opt 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/pbxt/t/lowercase_table_grant-master.opt 2009-08-31 11:07:44 +0000 @@ -0,0 +1 @@ +--lower_case_table_names === added file 'mysql-test/suite/pbxt/t/lowercase_table_qcache-master.opt' --- a/mysql-test/suite/pbxt/t/lowercase_table_qcache-master.opt 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/pbxt/t/lowercase_table_qcache-master.opt 2009-08-31 11:07:44 +0000 @@ -0,0 +1 @@ +--lower_case_table_names === added file 'mysql-test/suite/pbxt/t/lowercase_view-master.opt' --- a/mysql-test/suite/pbxt/t/lowercase_view-master.opt 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/pbxt/t/lowercase_view-master.opt 2009-08-31 11:07:44 +0000 @@ -0,0 +1 @@ +--lower_case_table_names=1 === modified file 'mysql-test/suite/pbxt/t/null.test' --- a/mysql-test/suite/pbxt/t/null.test 2009-04-02 20:36:52 +0000 +++ b/mysql-test/suite/pbxt/t/null.test 2009-08-17 15:57:58 +0000 @@ -61,9 +61,7 @@ drop table t1; # CREATE TABLE t1 (a varchar(16) NOT NULL default '', b smallint(6) NOT NULL default 0, c datetime NOT NULL default '0000-00-00 00:00:00', d smallint(6) NOT NULL default 0); INSERT INTO t1 SET a = "", d= "2003-01-14 03:54:55"; ---error 1048 UPDATE t1 SET d=1/NULL; ---error 1048 UPDATE t1 SET d=NULL; --error 1048 INSERT INTO t1 (a) values (null); === modified file 'mysql-test/suite/pbxt/t/pbxt_bugs.test' --- a/mysql-test/suite/pbxt/t/pbxt_bugs.test 2009-04-02 20:36:52 +0000 +++ b/mysql-test/suite/pbxt/t/pbxt_bugs.test 2009-08-17 15:57:58 +0000 @@ -926,7 +926,59 @@ LOAD DATA LOCAL INFILE 'suite/pbxt/t/loa SELECT * FROM t1 ORDER BY c1; DROP TABLE t1; +create table parent (id int primary key); +create table child (id int PRIMARY KEY, FOREIGN KEY (id) REFERENCES parent(id)); +insert into parent values (2), (3), (4); +insert into child values (3), (4); + +--error 1451 +delete ignore from parent; +--sorted_result +select * from parent; + +drop table child, parent; + +# bug 378222: Drop sakila causes error: Cannot delete or update a parent row: a foreign key constraint fails + +create schema test378222; +use test378222; +create table t1 (id int primary key); +create table t2 (id int primary key); +alter table t1 add constraint foreign key (id) references t2 (id); +alter table t2 add constraint foreign key (id) references t1 (id); +drop schema test378222; + +create schema test378222a; +create schema test378222b; +create table test378222a.t1 (id int primary key); +create table test378222b.t2 (id int primary key); +alter table test378222a.t1 add constraint foreign key (id) references test378222b.t2 (id); +alter table test378222b.t2 add constraint foreign key (id) references test378222a.t1 (id); +set foreign_key_checks = 1; +--error 1217 +drop schema test378222a; +--error 1217 +drop schema test378222b; +set foreign_key_checks = 0; +drop schema test378222a; +drop schema test378222b; +set foreign_key_checks = 1; +use test; + +# bug 369086: Incosistent/Incorrect Truncate behavior +CREATE TABLE t1(c1 TINYINT AUTO_INCREMENT NULL KEY ) AUTO_INCREMENT=10; +SHOW CREATE TABLE t1; +INSERT INTO t1 VALUES(null); +INSERT INTO t1 VALUES(null); +INSERT INTO t1 VALUES(null); +SELECT * FROM t1; +TRUNCATE TABLE t1; +INSERT INTO t1 VALUES(null); +SELECT * FROM t1; +DROP TABLE t1; + --disable_query_log + DROP TABLE t2, t5; drop database pbxt; --enable_query_log === modified file 'mysql-test/suite/pbxt/t/rename.test' --- a/mysql-test/suite/pbxt/t/rename.test 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/t/rename.test 2009-08-17 15:57:58 +0000 @@ -63,7 +63,28 @@ connection con2; # Wait for the the tables to be renamed # i.e the query below succeds let $query= select * from t2, t4; -source include/wait_for_query_to_suceed.inc; +# source include/wait_for_query_to_suceed.inc; +let $counter= 100; + +disable_abort_on_error; +disable_query_log; +disable_result_log; +eval $query; +while ($mysql_errno) +{ + eval $query; + sleep 0.1; + dec $counter; + + if (!$counter) + { + die("Waited too long for query to suceed"); + } +} +enable_abort_on_error; +enable_query_log; +enable_result_log; + show tables; === modified file 'mysql-test/suite/pbxt/t/schema.test' --- a/mysql-test/suite/pbxt/t/schema.test 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/t/schema.test 2009-08-17 15:57:58 +0000 @@ -10,5 +10,12 @@ drop database if exists mysqltest1; create schema foo; show create schema foo; +# force PBXT schema to be created +create table t1 (id int) engine=pbxt; show schemas; drop schema foo; + +--disable_query_log +drop table if exists t1; +drop database pbxt; +--enable_query_log === modified file 'mysql-test/suite/pbxt/t/type_enum.test' --- a/mysql-test/suite/pbxt/t/type_enum.test 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/t/type_enum.test 2009-08-17 15:57:58 +0000 @@ -153,12 +153,19 @@ create table t1(russian_deviant enum('E' show create table t1; drop table t1; -# ER_WRONG_FIELD_TERMINATORS ---error 1005 +# the following create statement sometimes fails like it would if NO_BACKSLASH_ESCAPES sql mode was on, +# we check sql mode here +select @@SESSION.sql_mode; +select @@GLOBAL.sql_mode; + +## ER_WRONG_FIELD_TERMINATORS +#--error 1005 create table t1(exhausting_charset enum('ABCDEFGHIJKLMNOPQRSTUVWXYZ',' !"','#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~','xx\','yy\�','zz��')); +drop table t1; + --disable_query_log drop database pbxt; --enable_query_log === added file 'mysql-test/suite/pbxt/t/udf-master.opt' --- a/mysql-test/suite/pbxt/t/udf-master.opt 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/pbxt/t/udf-master.opt 2009-08-31 11:07:44 +0000 @@ -0,0 +1 @@ +$UDF_EXAMPLE_LIB_OPT === modified file 'mysql-test/suite/pbxt/t/union.test' --- a/mysql-test/suite/pbxt/t/union.test 2009-04-02 10:03:14 +0000 +++ b/mysql-test/suite/pbxt/t/union.test 2009-08-17 15:57:58 +0000 @@ -802,6 +802,10 @@ drop tables t1,t2,t3; # exceeds mediumtext maximum length # +SELECT @tmp_max:= @@global.max_allowed_packet; +SET @@global.max_allowed_packet=25000000; +# switching connection to allow the new max_allowed_packet take effect +--connect (newconn, localhost, root,,) CREATE TABLE t1 (a mediumtext); CREATE TABLE t2 (b varchar(20)); INSERT INTO t1 VALUES ('a'); @@ -823,6 +827,9 @@ INSERT INTO t1 VALUES ('a'); CREATE TABLE t3 SELECT REPEAT(a,2) AS a FROM t1 UNION SELECT b FROM t2; SHOW CREATE TABLE t3; DROP TABLES t1,t2,t3; +--connection default +SET @@global.max_allowed_packet:= @tmp_max; +--disconnect newconn # # Bug #10032 Bug in parsing UNION with ORDER BY when one node does not use FROM === modified file 'storage/pbxt/ChangeLog' --- a/storage/pbxt/ChangeLog 2009-04-02 11:49:57 +0000 +++ b/storage/pbxt/ChangeLog 2009-08-18 07:46:53 +0000 @@ -1,7 +1,75 @@ PBXT Release Notes ================== -------- 1.0.08 RC - Not yet released +------- 1.0.08c RC2 - 2009-08-18 + +RN266: Updated BLOB streaming glue, used with the PBMS engine. The glue code is now identical to the version of "1.0.08-rc-pbms" version of PBXT available from http://blobstreaming.org/download. + +RN265: Changes the sequential reading of data log files to skip gaps, instead of returning EOF. This ensures that extended data records are preserved even when something goes wrong with the way the file is written. + +RN264: Fixed a bug that cased an "Data log not found" error after an out of disk space error on a log file. This bug is similar to RN262 in that it allows "gaps" to appear in the data logs. + +RN263: Updated xtstat to compile on Windows/MS Visual C++. + +RN262: Merged changes for PBMS version 0.5.09. + +RN261: Concerning bug #377788: Cannot find index for FK. Fixed buffer overflow which occurred when the error was reported. + +RN260: Fixed bug #377788: Cannot find index for FK. PBXT now correctly uses prefix of an index to support FK references (e.g. if key = (c1, c2) then an index on (c1, c2, c3) will work). Also fixed buffer overflow, which occurred when reporting the error. + +RN259: Fixed bug #309424: xtstat doesn't use my.cnf. You can now add an [xtstat] section to my.cnf, for use with xtstat. + +RN258: updated xt_p_join implementation for Windows to check if a thread has already exited or has not yet started + +RN257: Removed false assertion that could fail during restore if a transaction log page was zero-filled + +RN256: Update datalog eof pointer only if write opearions were sucessful + +RN255: Added re-allocation of of filemap if allocating the of the new map failed. This often happens if there's not enough space on disk. + +RN254: When a table with a corrupted index is detected, PBXT creates a file called 'repair-pending' in the pbxt directory, with the name of the table in it. Each table in the file is listed on a line by itself (the last line has no trailing \n). When the table is repaired (using the REPAIR TABLE command), this entry is removed from the file. + +RN253: Use fcntl(F_FULLFSYNC) instead of fsync on platforms that support it. Improper fsync operation was presumably the reason of index corruption on Mac OS X. + +RN252: Fixed bug #368692: PBXT not reporting data size correctly in information_schema. + +------- 1.0.08 RC2 - 2009-06-30 + +RN251: A Windows-specific test update, also removed false assertion that failed on Windows. + +RN250: Fixed a bug that caused recovery to fail when the transaction log ID exceeded 255. The problem was a checksum failed in the log record. + +RN249: Fixed bug #313176: Test case timeout. This happened because record cache pages where not properly freed and as soon as cache filled up the performacne degraded. + +RN248: PBXT now compiles and runs with MySQL 5.1.35. All tests pass. + +RN247: Fixed bug #369086: Incosistent/Incorrect Truncate behavior + +RN246: Fixed bug #378222: Drop sakila causes error: Cannot delete or update a parent row: a foreign key constraint fails + +RN245: Fixed bug #379315: Inconsistent behavior of DELETE IGNORE and FK constraint. + +RN244: Fixed a recovery problem: during the recovery of "record modified" action the table was updated before the old index entries were removed; then the xres_remove_index_entries was supplied the new record which lead to incorrect index update. + +RN243: Fixed a bug that caused a recovery failure if partitioned pbxt tables where present. This happended because the recovery used a MySQL function to open tables and the PBXT handler was not yet registered + +RN242: Fixed a bug that caused a deadlock if pbxt initialization failed. This happened because pbxt ceanup was done from pbxt_init() with PLUGIN_lock being held by MySQL which lead to a deadlock in the freeer thread + +RN241: Fixed a heap corruption bug (writing to a freed memory location). It happened only when memory mapped files were used leading to heap inconsistency and program crash or termination by heap checker. Likely to happen right after or during DROP TABLE but possible in other cases too. + +RN240: Load the record cache on read when no using memory mapped files. + +RN239: Added PBXT variable pbxt_max_threads. This is the maximum number of threads that can be created PBXT. By default this value is set to 0 which means the number of threads is derived from the MySQL variable max_connections. The value used is max_connections+7. Under Drizzle the default value is 500. + +RN238: Added an option to wait for the sweeper to clean up old transactions on a particular connection. This prevents the sweeper from getting too far behind. + +RN237: Added an option to lazy delete fixed length index entries. This means the index entries are just marked for deletion, instead of removing the items from the index page. This has the advantage that an exclusive lock is not always required for deletion. + +RN236: Fixed bug #349177: a bug in configure.in script. + +RN235: Fixed bug 349176: a compiler warning. + +RN234: Completed Drizzle integration. All Drizzle tests now run with PBXT. RN233: Fixed bugs which occur when PBXT is used together with PBMS (BLOB Streaming engine). === modified file 'storage/pbxt/src/Makefile.am' --- a/storage/pbxt/src/Makefile.am 2009-05-09 04:01:53 +0000 +++ b/storage/pbxt/src/Makefile.am 2009-08-31 11:07:44 +0000 @@ -19,7 +19,7 @@ noinst_HEADERS = bsearch_xt.h cache_xt. datadic_xt.h datalog_xt.h filesys_xt.h hashtab_xt.h \ ha_pbxt.h heap_xt.h index_xt.h linklist_xt.h \ memory_xt.h myxt_xt.h pthread_xt.h restart_xt.h \ - streaming_xt.h sortedlist_xt.h strutil_xt.h \ + pbms_enabled.h sortedlist_xt.h strutil_xt.h \ tabcache_xt.h table_xt.h trace_xt.h thread_xt.h \ util_xt.h xaction_xt.h xactlog_xt.h lock_xt.h \ systab_xt.h ha_xtsys.h discover_xt.h \ @@ -30,7 +30,7 @@ libpbxt_la_SOURCES = bsearch_xt.cc cache datadic_xt.cc datalog_xt.cc filesys_xt.cc hashtab_xt.cc \ ha_pbxt.cc heap_xt.cc index_xt.cc linklist_xt.cc \ memory_xt.cc myxt_xt.cc pthread_xt.cc restart_xt.cc \ - streaming_xt.cc sortedlist_xt.cc strutil_xt.cc \ + pbms_enabled.cc sortedlist_xt.cc strutil_xt.cc \ tabcache_xt.cc table_xt.cc trace_xt.cc thread_xt.cc \ systab_xt.cc ha_xtsys.cc discover_xt.cc \ util_xt.cc xaction_xt.cc xactlog_xt.cc lock_xt.cc locklist_xt.cc === modified file 'storage/pbxt/src/cache_xt.cc' --- a/storage/pbxt/src/cache_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/cache_xt.cc 2009-08-17 11:12:36 +0000 @@ -23,6 +23,10 @@ #include "xt_config.h" +#ifdef DRIZZLED +#include <bitset> +#endif + #ifndef XT_WIN #include <unistd.h> #endif @@ -51,17 +55,22 @@ #define IDX_CAC_SEGMENT_COUNT ((off_t) 1 << XT_INDEX_CACHE_SEGMENT_SHIFTS) #define IDX_CAC_SEGMENT_MASK (IDX_CAC_SEGMENT_COUNT - 1) -//#define IDX_USE_SPINRWLOCK -#define IDX_USE_RWMUTEX +#ifdef XT_NO_ATOMICS +#define IDX_CAC_USE_PTHREAD_RW +#else +//#define IDX_CAC_USE_RWMUTEX //#define IDX_CAC_USE_PTHREAD_RW +//#define IDX_USE_SPINXSLOCK +#define IDX_CAC_USE_XSMUTEX +#endif -#ifdef IDX_CAC_USE_FASTWRLOCK -#define IDX_CAC_LOCK_TYPE XTFastRWLockRec -#define IDX_CAC_INIT_LOCK(s, i) xt_fastrwlock_init(s, &(i)->cs_lock) -#define IDX_CAC_FREE_LOCK(s, i) xt_fastrwlock_free(s, &(i)->cs_lock) -#define IDX_CAC_READ_LOCK(i, o) xt_fastrwlock_slock(&(i)->cs_lock, (o)) -#define IDX_CAC_WRITE_LOCK(i, o) xt_fastrwlock_xlock(&(i)->cs_lock, (o)) -#define IDX_CAC_UNLOCK(i, o) xt_fastrwlock_unlock(&(i)->cs_lock, (o)) +#ifdef IDX_CAC_USE_XSMUTEX +#define IDX_CAC_LOCK_TYPE XTXSMutexRec +#define IDX_CAC_INIT_LOCK(s, i) xt_xsmutex_init_with_autoname(s, &(i)->cs_lock) +#define IDX_CAC_FREE_LOCK(s, i) xt_xsmutex_free(s, &(i)->cs_lock) +#define IDX_CAC_READ_LOCK(i, o) xt_xsmutex_slock(&(i)->cs_lock, (o)->t_id) +#define IDX_CAC_WRITE_LOCK(i, o) xt_xsmutex_xlock(&(i)->cs_lock, (o)->t_id) +#define IDX_CAC_UNLOCK(i, o) xt_xsmutex_unlock(&(i)->cs_lock, (o)->t_id) #elif defined(IDX_CAC_USE_PTHREAD_RW) #define IDX_CAC_LOCK_TYPE xt_rwlock_type #define IDX_CAC_INIT_LOCK(s, i) xt_init_rwlock(s, &(i)->cs_lock) @@ -69,13 +78,20 @@ #define IDX_CAC_READ_LOCK(i, o) xt_slock_rwlock_ns(&(i)->cs_lock) #define IDX_CAC_WRITE_LOCK(i, o) xt_xlock_rwlock_ns(&(i)->cs_lock) #define IDX_CAC_UNLOCK(i, o) xt_unlock_rwlock_ns(&(i)->cs_lock) -#elif defined(IDX_USE_RWMUTEX) +#elif defined(IDX_CAC_USE_RWMUTEX) #define IDX_CAC_LOCK_TYPE XTRWMutexRec #define IDX_CAC_INIT_LOCK(s, i) xt_rwmutex_init_with_autoname(s, &(i)->cs_lock) #define IDX_CAC_FREE_LOCK(s, i) xt_rwmutex_free(s, &(i)->cs_lock) #define IDX_CAC_READ_LOCK(i, o) xt_rwmutex_slock(&(i)->cs_lock, (o)->t_id) #define IDX_CAC_WRITE_LOCK(i, o) xt_rwmutex_xlock(&(i)->cs_lock, (o)->t_id) #define IDX_CAC_UNLOCK(i, o) xt_rwmutex_unlock(&(i)->cs_lock, (o)->t_id) +#elif defined(IDX_CAC_USE_SPINXSLOCK) +#define IDX_CAC_LOCK_TYPE XTSpinXSLockRec +#define IDX_CAC_INIT_LOCK(s, i) xt_spinxslock_init_with_autoname(s, &(i)->cs_lock) +#define IDX_CAC_FREE_LOCK(s, i) xt_spinxslock_free(s, &(i)->cs_lock) +#define IDX_CAC_READ_LOCK(i, s) xt_spinxslock_slock(&(i)->cs_lock, (s)->t_id) +#define IDX_CAC_WRITE_LOCK(i, s) xt_spinxslock_xlock(&(i)->cs_lock, (s)->t_id) +#define IDX_CAC_UNLOCK(i, s) xt_spinxslock_unlock(&(i)->cs_lock, (s)->t_id) #endif #define ID_HANDLE_USE_SPINLOCK @@ -308,7 +324,8 @@ xtPublic XTIndHandlePtr xt_ind_get_handl hs = &ind_cac_globals.cg_handle_slot[iref->ir_block->cb_address % XT_HANDLE_SLOTS]; - ASSERT_NS(iref->ir_ulock == XT_UNLOCK_READ); + ASSERT_NS(iref->ir_xlock == FALSE); + ASSERT_NS(iref->ir_updated == FALSE); ID_HANDLE_LOCK(&hs->hs_handles_lock); #ifdef CHECK_HANDLE_STRUCTS ic_check_handle_structs(); @@ -337,10 +354,10 @@ xtPublic XTIndHandlePtr xt_ind_get_handl * at least an Slock on the index. * So this excludes anyone who is reading * cb_handle_count in the index. - * (all cache block writers, and a freeer). + * (all cache block writers, and the freeer). * * The increment is safe because I have the list - * lock, which is required by anyone else + * lock (hs_handles_lock), which is required by anyone else * who increments or decrements this value. */ iref->ir_block->cb_handle_count++; @@ -396,8 +413,11 @@ xtPublic void xt_ind_release_handle(XTIn xblock = seg->cs_hash_table[hash_idx]; while (xblock) { if (block == xblock) { - /* Found the block... */ - xt_atomicrwlock_xlock(&block->cb_lock, thread->t_id); + /* Found the block... + * {HANDLE-COUNT-SLOCK} + * 04.05.2009, changed to slock. + */ + XT_IPAGE_READ_LOCK(&block->cb_lock); goto block_found; } xblock = xblock->cb_next; @@ -431,7 +451,18 @@ xtPublic void xt_ind_release_handle(XTIn /* {HANDLE-COUNT-USAGE} * This is safe here because I have excluded * all readers by taking an Xlock on the - * cache block. + * cache block (CHANGED - see below). + * + * {HANDLE-COUNT-SLOCK} + * 04.05.2009, changed to slock. + * Should be OK, because: + * A have a lock on the list lock (hs_handles_lock), + * which prevents concurrent updates to cb_handle_count. + * + * I have also have a read lock on the cache block + * but not a lock on the index. As a result, we cannot + * excluded all index writers (and readers of + * cb_handle_count. */ block->cb_handle_count--; } @@ -466,7 +497,7 @@ xtPublic void xt_ind_release_handle(XTIn ID_HANDLE_UNLOCK(&hs->hs_handles_lock); if (block) - xt_atomicrwlock_unlock(&block->cb_lock, TRUE); + XT_IPAGE_UNLOCK(&block->cb_lock, FALSE); } /* Call this function before a referenced cache block is modified! @@ -482,17 +513,28 @@ xtPublic xtBool xt_ind_copy_on_write(XTI hs = &ind_cac_globals.cg_handle_slot[iref->ir_block->cb_address % XT_HANDLE_SLOTS]; + ID_HANDLE_LOCK(&hs->hs_handles_lock); + /* {HANDLE-COUNT-USAGE} * This is only called by updaters of this index block, or * the free which holds an Xlock on the index block. - * * These are all mutually exclusive for the index block. + * + * {HANDLE-COUNT-SLOCK} + * Do this check again, after we have the list lock (hs_handles_lock). + * There is a small chance that the count has changed, since we last + * checked because xt_ind_release_handle() only holds + * an slock on the index page. + * + * An updater can sometimes have a XLOCK on the index and an slock + * on the cache block. In this case xt_ind_release_handle() + * could have run through. */ - ASSERT_NS(iref->ir_block->cb_handle_count); - if (!iref->ir_block->cb_handle_count) + if (!iref->ir_block->cb_handle_count) { + ID_HANDLE_UNLOCK(&hs->hs_handles_lock); return OK; + } - ID_HANDLE_LOCK(&hs->hs_handles_lock); #ifdef CHECK_HANDLE_STRUCTS ic_check_handle_structs(); #endif @@ -609,7 +651,7 @@ xtPublic void xt_ind_init(XTThreadPtr se #endif for (u_int i=0; i<ind_cac_globals.cg_block_count; i++) { - xt_atomicrwlock_init_with_autoname(self, &block->cb_lock); + XT_IPAGE_INIT_LOCK(self, &block->cb_lock); block->cb_state = IDX_CAC_BLOCK_FREE; block->cb_next = ind_cac_globals.cg_free_list; #ifdef XT_USE_DIRECT_IO_ON_INDEX @@ -836,10 +878,10 @@ static xtBool ind_free_block(XTOpenTable while (xblock) { if (block == xblock) { /* Found the block... */ - xt_atomicrwlock_xlock(&block->cb_lock, ot->ot_thread->t_id); + XT_IPAGE_WRITE_LOCK(&block->cb_lock, ot->ot_thread->t_id); if (block->cb_state != IDX_CAC_BLOCK_CLEAN) { /* This block cannot be freeed: */ - xt_atomicrwlock_unlock(&block->cb_lock, TRUE); + XT_IPAGE_UNLOCK(&block->cb_lock, TRUE); IDX_CAC_UNLOCK(seg, ot->ot_thread); #ifdef DEBUG_CHECK_IND_CACHE xt_ind_check_cache(NULL); @@ -878,11 +920,12 @@ static xtBool ind_free_block(XTOpenTable if (block->cb_handle_count) { XTIndReferenceRec iref; - iref.ir_ulock = XT_UNLOCK_WRITE; + iref.ir_xlock = TRUE; + iref.ir_updated = FALSE; iref.ir_block = block; iref.ir_branch = (XTIdxBranchDPtr) block->cb_data; if (!xt_ind_copy_on_write(&iref)) { - xt_atomicrwlock_unlock(&block->cb_lock, TRUE); + XT_IPAGE_UNLOCK(&block->cb_lock, TRUE); return FALSE; } } @@ -918,7 +961,7 @@ static xtBool ind_free_block(XTOpenTable IDX_TRACE("%d- f%x\n", (int) XT_NODE_ID(address), (int) XT_GET_DISK_2(block->cb_data)); /* Unlock BEFORE the block is reused! */ - xt_atomicrwlock_unlock(&block->cb_lock, TRUE); + XT_IPAGE_UNLOCK(&block->cb_lock, TRUE); xt_unlock_mutex_ns(&ind_cac_globals.cg_lock); @@ -1001,7 +1044,7 @@ static u_int ind_cac_free_lru_blocks(XTO * Fetch the block. Note, if we are about to write the block * then there is no need to read it from disk! */ -static XTIndBlockPtr ind_cac_fetch(XTOpenTablePtr ot, xtIndexNodeID address, DcSegmentPtr *ret_seg, xtBool read_data) +static XTIndBlockPtr ind_cac_fetch(XTOpenTablePtr ot, XTIndexPtr ind, xtIndexNodeID address, DcSegmentPtr *ret_seg, xtBool read_data) { register XTOpenFilePtr file = ot->ot_ind_file; register XTIndBlockPtr block, new_block; @@ -1110,6 +1153,7 @@ static XTIndBlockPtr ind_cac_fetch(XTOpe new_block->cb_state = IDX_CAC_BLOCK_CLEAN; new_block->cb_handle_count = 0; new_block->cp_flush_seq = 0; + new_block->cp_del_count = 0; new_block->cb_dirty_next = NULL; new_block->cb_dirty_prev = NULL; @@ -1172,6 +1216,13 @@ static XTIndBlockPtr ind_cac_fetch(XTOpe #endif xt_unlock_mutex_ns(&dcg->cg_lock); + /* {LAZY-DEL-INDEX-ITEMS} + * Conditionally count the number of deleted entries in the index: + * We do this before other threads can read the block. + */ + if (ind->mi_lazy_delete && read_data) + xt_ind_count_deleted_items(ot->ot_table, ind, block); + /* Add to the hash table: */ block->cb_next = seg->cs_hash_table[hash_idx]; seg->cs_hash_table[hash_idx] = block; @@ -1221,10 +1272,10 @@ xtPublic xtBool xt_ind_write(XTOpenTable XTIndBlockPtr block; DcSegmentPtr seg; - if (!(block = ind_cac_fetch(ot, address, &seg, FALSE))) + if (!(block = ind_cac_fetch(ot, ind, address, &seg, FALSE))) return FAILED; - xt_atomicrwlock_xlock(&block->cb_lock, ot->ot_thread->t_id); + XT_IPAGE_WRITE_LOCK(&block->cb_lock, ot->ot_thread->t_id); ASSERT_NS(block->cb_state == IDX_CAC_BLOCK_CLEAN || block->cb_state == IDX_CAC_BLOCK_DIRTY); memcpy(block->cb_data, data, size); block->cp_flush_seq = ot->ot_table->tab_ind_flush_seq; @@ -1239,7 +1290,7 @@ xtPublic xtBool xt_ind_write(XTOpenTable xt_spinlock_unlock(&ind->mi_dirty_lock); block->cb_state = IDX_CAC_BLOCK_DIRTY; } - xt_atomicrwlock_unlock(&block->cb_lock, TRUE); + XT_IPAGE_UNLOCK(&block->cb_lock, TRUE); IDX_CAC_UNLOCK(seg, ot->ot_thread); #ifdef XT_TRACK_INDEX_UPDATES ot->ot_ind_changed++; @@ -1259,10 +1310,10 @@ xtPublic xtBool xt_ind_write_cache(XTOpe return FAILED; if (block) { - xt_atomicrwlock_xlock(&block->cb_lock, ot->ot_thread->t_id); + XT_IPAGE_WRITE_LOCK(&block->cb_lock, ot->ot_thread->t_id); ASSERT_NS(block->cb_state == IDX_CAC_BLOCK_CLEAN || block->cb_state == IDX_CAC_BLOCK_DIRTY); memcpy(block->cb_data, data, size); - xt_atomicrwlock_unlock(&block->cb_lock, TRUE); + XT_IPAGE_UNLOCK(&block->cb_lock, TRUE); IDX_CAC_UNLOCK(seg, ot->ot_thread); } @@ -1277,7 +1328,7 @@ xtPublic xtBool xt_ind_clean(XTOpenTable if (!ind_cac_get(ot, address, &seg, &block)) return FAILED; if (block) { - xt_atomicrwlock_xlock(&block->cb_lock, ot->ot_thread->t_id); + XT_IPAGE_WRITE_LOCK(&block->cb_lock, ot->ot_thread->t_id); ASSERT_NS(block->cb_state == IDX_CAC_BLOCK_CLEAN || block->cb_state == IDX_CAC_BLOCK_DIRTY); if (block->cb_state == IDX_CAC_BLOCK_DIRTY) { @@ -1293,7 +1344,7 @@ xtPublic xtBool xt_ind_clean(XTOpenTable xt_spinlock_unlock(&ind->mi_dirty_lock); block->cb_state = IDX_CAC_BLOCK_CLEAN; } - xt_atomicrwlock_unlock(&block->cb_lock, TRUE); + XT_IPAGE_UNLOCK(&block->cb_lock, TRUE); IDX_CAC_UNLOCK(seg, ot->ot_thread); } @@ -1301,29 +1352,33 @@ xtPublic xtBool xt_ind_clean(XTOpenTable return OK; } -xtPublic xtBool xt_ind_read_bytes(XTOpenTablePtr ot, xtIndexNodeID address, size_t size, xtWord1 *data) +xtPublic xtBool xt_ind_read_bytes(XTOpenTablePtr ot, XTIndexPtr ind, xtIndexNodeID address, size_t size, xtWord1 *data) { XTIndBlockPtr block; DcSegmentPtr seg; - if (!(block = ind_cac_fetch(ot, address, &seg, TRUE))) + if (!(block = ind_cac_fetch(ot, ind, address, &seg, TRUE))) return FAILED; - xt_atomicrwlock_slock(&block->cb_lock); + XT_IPAGE_READ_LOCK(&block->cb_lock); memcpy(data, block->cb_data, size); - xt_atomicrwlock_unlock(&block->cb_lock, FALSE); + XT_IPAGE_UNLOCK(&block->cb_lock, FALSE); IDX_CAC_UNLOCK(seg, ot->ot_thread); return OK; } -xtPublic xtBool xt_ind_fetch(XTOpenTablePtr ot, xtIndexNodeID address, XTPageLockType ltype, XTIndReferencePtr iref) +xtPublic xtBool xt_ind_fetch(XTOpenTablePtr ot, XTIndexPtr ind, xtIndexNodeID address, XTPageLockType ltype, XTIndReferencePtr iref) { register XTIndBlockPtr block; DcSegmentPtr seg; xtWord2 branch_size; + xtBool xlock = FALSE; - ASSERT_NS(iref->ir_ulock == XT_UNLOCK_NONE); - if (!(block = ind_cac_fetch(ot, address, &seg, TRUE))) +#ifdef DEBUG + ASSERT_NS(iref->ir_xlock == 2); + ASSERT_NS(iref->ir_xlock == 2); +#endif + if (!(block = ind_cac_fetch(ot, ind, address, &seg, TRUE))) return NULL; branch_size = XT_GET_DISK_2(((XTIdxBranchDPtr) block->cb_data)->tb_size_2); @@ -1333,21 +1388,50 @@ xtPublic xtBool xt_ind_fetch(XTOpenTable return FAILED; } - if (ltype == XT_XLOCK_LEAF) { - if (XT_IS_NODE(branch_size)) - ltype = XT_LOCK_READ; - else - ltype = XT_LOCK_WRITE; + switch (ltype) { + case XT_LOCK_READ: + break; + case XT_LOCK_WRITE: + xlock = TRUE; + break; + case XT_XLOCK_LEAF: + if (!XT_IS_NODE(branch_size)) + xlock = TRUE; + break; + case XT_XLOCK_DEL_LEAF: + if (!XT_IS_NODE(branch_size)) { + if (ot->ot_table->tab_dic.dic_no_lazy_delete) + xlock = TRUE; + else { + /* + * {LAZY-DEL-INDEX-ITEMS} + * + * We are fetch a page for delete purpose. + * we decide here if we plan to do a lazy delete, + * Or if we plan to compact the node. + * + * A lazy delete just requires a shared lock. + * + */ + if (ind->mi_lazy_delete) { + /* If the number of deleted items is greater than + * half of the number of times that can fit in the + * page, then we will compact the node. + */ + if (!xt_idx_lazy_delete_on_leaf(ind, block, XT_GET_INDEX_BLOCK_LEN(branch_size))) + xlock = TRUE; + } + else + xlock = TRUE; + } + } + break; } - if (ltype == XT_LOCK_WRITE) { - xt_atomicrwlock_xlock(&block->cb_lock, ot->ot_thread->t_id); - iref->ir_ulock = XT_UNLOCK_WRITE; - } - else { - xt_atomicrwlock_slock(&block->cb_lock); - iref->ir_ulock = XT_UNLOCK_READ; - } + if ((iref->ir_xlock = xlock)) + XT_IPAGE_WRITE_LOCK(&block->cb_lock, ot->ot_thread->t_id); + else + XT_IPAGE_READ_LOCK(&block->cb_lock); IDX_CAC_UNLOCK(seg, ot->ot_thread); @@ -1358,18 +1442,31 @@ xtPublic xtBool xt_ind_fetch(XTOpenTable * As a result, we need to pass a pointer to both the * cache block and the cache block data: */ + iref->ir_updated = FALSE; iref->ir_block = block; iref->ir_branch = (XTIdxBranchDPtr) block->cb_data; return OK; } -xtPublic xtBool xt_ind_release(XTOpenTablePtr ot, XTIndexPtr ind, XTPageUnlockType XT_UNUSED(utype), XTIndReferencePtr iref) +xtPublic xtBool xt_ind_release(XTOpenTablePtr ot, XTIndexPtr ind, XTPageUnlockType XT_NDEBUG_UNUSED(utype), XTIndReferencePtr iref) { register XTIndBlockPtr block; block = iref->ir_block; - if (utype == XT_UNLOCK_R_UPDATE || utype == XT_UNLOCK_W_UPDATE) { +#ifdef DEBUG + ASSERT_NS(iref->ir_xlock != 2); + ASSERT_NS(iref->ir_updated != 2); + if (iref->ir_updated) + ASSERT_NS(utype == XT_UNLOCK_R_UPDATE || utype == XT_UNLOCK_W_UPDATE); + else + ASSERT_NS(utype == XT_UNLOCK_READ || utype == XT_UNLOCK_WRITE); + if (iref->ir_xlock) + ASSERT_NS(utype == XT_UNLOCK_WRITE || utype == XT_UNLOCK_W_UPDATE); + else + ASSERT_NS(utype == XT_UNLOCK_READ || utype == XT_UNLOCK_R_UPDATE); +#endif + if (iref->ir_updated) { /* The page was update: */ ASSERT_NS(block->cb_state == IDX_CAC_BLOCK_CLEAN || block->cb_state == IDX_CAC_BLOCK_DIRTY); block->cp_flush_seq = ot->ot_table->tab_ind_flush_seq; @@ -1386,16 +1483,10 @@ xtPublic xtBool xt_ind_release(XTOpenTab } } + XT_IPAGE_UNLOCK(&block->cb_lock, iref->ir_xlock); #ifdef DEBUG - if (utype == XT_UNLOCK_W_UPDATE) - utype = XT_UNLOCK_WRITE; - else if (utype == XT_UNLOCK_R_UPDATE) - utype = XT_UNLOCK_READ; - ASSERT_NS(iref->ir_ulock == utype); -#endif - xt_atomicrwlock_unlock(&block->cb_lock, iref->ir_ulock == XT_UNLOCK_WRITE ? TRUE : FALSE); -#ifdef DEBUG - iref->ir_ulock = XT_UNLOCK_NONE; + iref->ir_xlock = 2; + iref->ir_updated = 2; #endif return OK; } @@ -1484,24 +1575,3 @@ xtPublic void xt_ind_unreserve(XTOpenTab xt_ind_free_reserved(ot); } -xtPublic void xt_load_indices(XTThreadPtr self, XTOpenTablePtr ot) -{ - register XTTableHPtr tab = ot->ot_table; - register XTIndBlockPtr block; - DcSegmentPtr seg; - xtIndexNodeID id; - - xt_lock_mutex_ns(&tab->tab_ind_flush_lock); - - for (id=1; id < XT_NODE_ID(tab->tab_ind_eof); id++) { - if (!(block = ind_cac_fetch(ot, id, &seg, TRUE))) { - xt_unlock_mutex_ns(&tab->tab_ind_flush_lock); - xt_throw(self); - } - IDX_CAC_UNLOCK(seg, ot->ot_thread); - } - - xt_unlock_mutex_ns(&tab->tab_ind_flush_lock); -} - - === modified file 'storage/pbxt/src/cache_xt.h' --- a/storage/pbxt/src/cache_xt.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/cache_xt.h 2009-08-17 11:12:36 +0000 @@ -45,8 +45,46 @@ struct XTIdxReadBuffer; #define IDX_CAC_BLOCK_CLEAN 1 #define IDX_CAC_BLOCK_DIRTY 2 -typedef enum XTPageLockType { XT_LOCK_READ, XT_LOCK_WRITE, XT_XLOCK_LEAF }; -typedef enum XTPageUnlockType { XT_UNLOCK_NONE, XT_UNLOCK_READ, XT_UNLOCK_WRITE, XT_UNLOCK_R_UPDATE, XT_UNLOCK_W_UPDATE }; +#ifdef XT_NO_ATOMICS +#define XT_IPAGE_USE_PTHREAD_RW +#else +//#define XT_IPAGE_USE_ATOMIC_RW +#define XT_IPAGE_USE_SPINXSLOCK +//#define XT_IPAGE_USE_SKEW_RW +#endif + +#ifdef XT_IPAGE_USE_ATOMIC_RW +#define XT_IPAGE_LOCK_TYPE XTAtomicRWLockRec +#define XT_IPAGE_INIT_LOCK(s, i) xt_atomicrwlock_init_with_autoname(s, i) +#define XT_IPAGE_FREE_LOCK(s, i) xt_atomicrwlock_free(s, i) +#define XT_IPAGE_READ_LOCK(i) xt_atomicrwlock_slock(i) +#define XT_IPAGE_WRITE_LOCK(i, o) xt_atomicrwlock_xlock(i, o) +#define XT_IPAGE_UNLOCK(i, x) xt_atomicrwlock_unlock(i, x) +#elif defined(XT_IPAGE_USE_PTHREAD_RW) +#define XT_IPAGE_LOCK_TYPE xt_rwlock_type +#define XT_IPAGE_INIT_LOCK(s, i) xt_init_rwlock(s, i) +#define XT_IPAGE_FREE_LOCK(s, i) xt_free_rwlock(i) +#define XT_IPAGE_READ_LOCK(i) xt_slock_rwlock_ns(i) +#define XT_IPAGE_WRITE_LOCK(i, s) xt_xlock_rwlock_ns(i) +#define XT_IPAGE_UNLOCK(i, x) xt_unlock_rwlock_ns(i) +#elif defined(XT_IPAGE_USE_SPINXSLOCK) +#define XT_IPAGE_LOCK_TYPE XTSpinXSLockRec +#define XT_IPAGE_INIT_LOCK(s, i) xt_spinxslock_init_with_autoname(s, i) +#define XT_IPAGE_FREE_LOCK(s, i) xt_spinxslock_free(s, i) +#define XT_IPAGE_READ_LOCK(i) xt_spinxslock_slock(i) +#define XT_IPAGE_WRITE_LOCK(i, o) xt_spinxslock_xlock(i, o) +#define XT_IPAGE_UNLOCK(i, x) xt_spinxslock_unlock(i, x) +#else // XT_IPAGE_USE_SKEW_RW +#define XT_IPAGE_LOCK_TYPE XTSkewRWLockRec +#define XT_IPAGE_INIT_LOCK(s, i) xt_skewrwlock_init_with_autoname(s, i) +#define XT_IPAGE_FREE_LOCK(s, i) xt_skewrwlock_free(s, i) +#define XT_IPAGE_READ_LOCK(i) xt_skewrwlock_slock(i) +#define XT_IPAGE_WRITE_LOCK(i, o) xt_skewrwlock_xlock(i, o) +#define XT_IPAGE_UNLOCK(i, x) xt_skewrwlock_unlock(i, x) +#endif + +enum XTPageLockType { XT_LOCK_READ, XT_LOCK_WRITE, XT_XLOCK_LEAF, XT_XLOCK_DEL_LEAF }; +enum XTPageUnlockType { XT_UNLOCK_NONE, XT_UNLOCK_READ, XT_UNLOCK_WRITE, XT_UNLOCK_R_UPDATE, XT_UNLOCK_W_UPDATE }; /* A block is X locked if it is being changed or freed. * A block is S locked if it is being read. @@ -64,10 +102,11 @@ typedef struct XTIndBlock { struct XTIndBlock *cb_mr_used; /* More recently used blocks. */ struct XTIndBlock *cb_lr_used; /* Less recently used blocks. */ /* Protected by cb_lock: */ - XTAtomicRWLockRec cb_lock; + XT_IPAGE_LOCK_TYPE cb_lock; xtWord1 cb_state; /* Block status. */ xtWord2 cb_handle_count; /* TRUE if this page is referenced by a handle. */ xtWord2 cp_flush_seq; + xtWord2 cp_del_count; /* Number of deleted entries. */ #ifdef XT_USE_DIRECT_IO_ON_INDEX xtWord1 *cb_data; #else @@ -76,16 +115,18 @@ typedef struct XTIndBlock { } XTIndBlockRec, *XTIndBlockPtr; typedef struct XTIndReference { - XTPageUnlockType ir_ulock; + xtBool ir_xlock; /* Set to TRUE if the cache block is X locked. */ + xtBool ir_updated; /* Set to TRUE if the cache block is updated. */ XTIndBlockPtr ir_block; XTIdxBranchDPtr ir_branch; } XTIndReferenceRec, *XTIndReferencePtr; typedef struct XTIndFreeBlock { + XTDiskValue1 if_zero1_1; /* Must be set to zero. */ + XTDiskValue1 if_zero2_1; /* Must be set to zero. */ XTDiskValue1 if_status_1; XTDiskValue1 if_unused1_1; - XTDiskValue2 if_unused2_2; - XTDiskValue4 if_unused3_4; + XTDiskValue4 if_unused2_4; XTDiskValue8 if_next_block_8; } XTIndFreeBlockRec, *XTIndFreeBlockPtr; @@ -116,14 +157,13 @@ xtInt8 xt_ind_get_size(); xtBool xt_ind_write(struct XTOpenTable *ot, XTIndexPtr ind, xtIndexNodeID offset, size_t size, xtWord1 *data); xtBool xt_ind_write_cache(struct XTOpenTable *ot, xtIndexNodeID offset, size_t size, xtWord1 *data); xtBool xt_ind_clean(struct XTOpenTable *ot, XTIndexPtr ind, xtIndexNodeID offset); -xtBool xt_ind_read_bytes(struct XTOpenTable *ot, xtIndexNodeID offset, size_t size, xtWord1 *data); +xtBool xt_ind_read_bytes(struct XTOpenTable *ot, XTIndexPtr ind, xtIndexNodeID offset, size_t size, xtWord1 *data); void xt_ind_check_cache(XTIndexPtr ind); xtBool xt_ind_reserve(struct XTOpenTable *ot, u_int count, XTIdxBranchDPtr not_this); void xt_ind_free_reserved(struct XTOpenTable *ot); void xt_ind_unreserve(struct XTOpenTable *ot); -void xt_load_indices(XTThreadPtr self, struct XTOpenTable *ot); -xtBool xt_ind_fetch(struct XTOpenTable *ot, xtIndexNodeID node, XTPageLockType ltype, XTIndReferencePtr iref); +xtBool xt_ind_fetch(struct XTOpenTable *ot, XTIndexPtr ind, xtIndexNodeID node, XTPageLockType ltype, XTIndReferencePtr iref); xtBool xt_ind_release(struct XTOpenTable *ot, XTIndexPtr ind, XTPageUnlockType utype, XTIndReferencePtr iref); void xt_ind_lock_handle(XTIndHandlePtr handle); === modified file 'storage/pbxt/src/ccutils_xt.cc' --- a/storage/pbxt/src/ccutils_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/ccutils_xt.cc 2009-08-17 11:12:36 +0000 @@ -29,7 +29,7 @@ #include "ccutils_xt.h" #include "bsearch_xt.h" -static int ccu_compare_object(XTThreadPtr XT_UNUSED(self), register const void XT_UNUSED(*thunk), register const void *a, register const void *b) +static int ccu_compare_object(XTThreadPtr XT_UNUSED(self), register const void *XT_UNUSED(thunk), register const void *a, register const void *b) { XTObject *obj_ptr = (XTObject *) b; === modified file 'storage/pbxt/src/database_xt.cc' --- a/storage/pbxt/src/database_xt.cc 2009-04-02 10:03:14 +0000 +++ b/storage/pbxt/src/database_xt.cc 2009-08-17 11:12:36 +0000 @@ -23,6 +23,10 @@ #include "xt_config.h" +#ifdef DRIZZLED +#include <bitset> +#endif + #include <string.h> #include <stdio.h> @@ -240,7 +244,7 @@ static void db_hash_free(XTThreadPtr sel xt_heap_release(self, (XTDatabaseHPtr) data); } -static int db_cmp_db_id(struct XTThread XT_UNUSED(*self), register const void XT_UNUSED(*thunk), register const void *a, register const void *b) +static int db_cmp_db_id(struct XTThread *XT_UNUSED(self), register const void *XT_UNUSED(thunk), register const void *a, register const void *b) { xtDatabaseID db_id = *((xtDatabaseID *) a); XTDatabaseHPtr *db_ptr = (XTDatabaseHPtr *) b; @@ -346,7 +350,7 @@ static void db_finalize(XTThreadPtr self } } -static void db_onrelease(XTThreadPtr self, void XT_UNUSED(*x)) +static void db_onrelease(XTThreadPtr self, void *XT_UNUSED(x)) { /* Signal threads waiting for exclusive use of the database: */ if (xt_db_open_databases) // The database may already be closed. @@ -612,7 +616,7 @@ xtPublic void xt_drop_database(XTThreadP xtPublic void xt_open_database(XTThreadPtr self, char *path, xtBool multi_path) { XTDatabaseHPtr db; - + /* We cannot get a database, without unusing the current * first. The reason is that the restart process will * partially set the current database! @@ -621,7 +625,7 @@ xtPublic void xt_open_database(XTThreadP db = xt_get_database(self, path, multi_path); pushr_(xt_heap_release, db); xt_use_database(self, db, XT_FOR_USER); - freer_(); // xt_heap_release(self, db); + freer_(); // xt_heap_release(self, db); } /* This function can only be called if you do not already have a database in @@ -638,6 +642,12 @@ xtPublic void xt_use_database(XTThreadPt xt_heap_reference(self, db); self->st_database = db; +#ifdef XT_WAIT_FOR_CLEANUP + self->st_last_xact = 0; + for (int i=0; i<XT_MAX_XACT_BEHIND; i++) { + self->st_prev_xact[i] = db->db_xn_curr_id; + } +#endif xt_xn_init_thread(self, what_for); } @@ -1117,15 +1127,18 @@ xtPublic void xt_db_return_table_to_pool XTDatabaseHPtr db = ot->ot_table->tab_db; xtBool flush_table = TRUE; + /* No open table returned to the pool should still + * have a cache handle! + */ + ASSERT_NS(!ot->ot_ind_rhandle); xt_lock_mutex_ns(&db->db_ot_pool.opt_lock); if (!(table_pool = db_get_open_table_pool(db, ot->ot_table->tab_id))) goto failed; if (table_pool->opt_locked && !table_pool->opt_flushing) { - table_pool->opt_total_open--; /* Table will be closed below: */ - if (table_pool->opt_total_open > 0) + if (table_pool->opt_total_open > 1) flush_table = FALSE; } else { @@ -1151,14 +1164,21 @@ xtPublic void xt_db_return_table_to_pool ot = NULL; } + if (ot) { + xt_unlock_mutex_ns(&db->db_ot_pool.opt_lock); + xt_close_table(ot, flush_table, FALSE); + + /* assume that table_pool cannot be invalidated in between as we have table_pool->opt_total_open > 0 */ + xt_lock_mutex_ns(&db->db_ot_pool.opt_lock); + table_pool->opt_total_open--; + } + db_free_open_table_pool(NULL, table_pool); if (!xt_broadcast_cond_ns(&db->db_ot_pool.opt_cond)) goto failed; xt_unlock_mutex_ns(&db->db_ot_pool.opt_lock); - if (ot) - xt_close_table(ot, flush_table, FALSE); - + return; failed: === modified file 'storage/pbxt/src/datadic_xt.cc' --- a/storage/pbxt/src/datadic_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/datadic_xt.cc 2009-08-18 07:46:53 +0000 @@ -26,6 +26,10 @@ #include "xt_config.h" +#ifdef DRIZZLED +#include <bitset> +#endif + #include <ctype.h> #include <errno.h> @@ -433,7 +437,7 @@ class XTTokenizer { XTToken *nextToken(XTThreadPtr self, c_char *keyword, XTToken *tk); }; -void ri_free_token(XTThreadPtr self __attribute__((unused)), XTToken *tk) +void ri_free_token(XTThreadPtr XT_UNUSED(self), XTToken *tk) { delete tk; } @@ -524,6 +528,13 @@ XTToken *XTTokenizer::nextToken(XTThread break; tkn_curr_pos++; } + /* TODO: Unless sql_mode == 'NO_BACKSLASH_ESCAPES'!!! */ + if (*tkn_curr_pos == '\\') { + if (*(tkn_curr_pos+1) == quote) { + if (quote == '"' || quote == '\'') + tkn_curr_pos++; + } + } tkn_curr_pos++; } @@ -639,7 +650,7 @@ class XTParseTable : public XTObject { int parseKeyAction(XTThreadPtr self); void parseCreateTable(XTThreadPtr self); void parseAddTableItem(XTThreadPtr self); - void parseQualifiedName(XTThreadPtr self, char *name); + void parseQualifiedName(XTThreadPtr self, char *parent_name, char *name); void parseTableName(XTThreadPtr self, bool alterTable); void parseExpression(XTThreadPtr self, bool allow_reserved); void parseBrackets(XTThreadPtr self); @@ -667,53 +678,53 @@ class XTParseTable : public XTObject { memset(&pt_sbuffer, 0, sizeof(XTStringBufferRec)); } - virtual void finalize(XTThreadPtr self __attribute__((unused))) { + virtual void finalize(XTThreadPtr XT_UNUSED(self)) { if (pt_tokenizer) delete pt_tokenizer; xt_sb_set_size(NULL, &pt_sbuffer, 0); } // Hooks to receive output from the parser: - virtual void setTableName(XTThreadPtr self __attribute__((unused)), char *name __attribute__((unused)), bool alterTable __attribute__((unused))) { + virtual void setTableName(XTThreadPtr XT_UNUSED(self), char *XT_UNUSED(name), bool XT_UNUSED(alterTable)) { } - virtual void addColumn(XTThreadPtr self __attribute__((unused)), char *col_name __attribute__((unused)), char *old_col_name __attribute__((unused))) { + virtual void addColumn(XTThreadPtr XT_UNUSED(self), char *XT_UNUSED(col_name), char *XT_UNUSED(old_col_name)) { } virtual void setDataType(XTThreadPtr self, char *cstring) { if (cstring) xt_free(self, cstring); } - virtual void setNull(XTThreadPtr self __attribute__((unused)), bool nullOK __attribute__((unused))) { + virtual void setNull(XTThreadPtr XT_UNUSED(self), bool XT_UNUSED(nullOK)) { } - virtual void setAutoInc(XTThreadPtr self __attribute__((unused)), bool autoInc __attribute__((unused))) { + virtual void setAutoInc(XTThreadPtr XT_UNUSED(self), bool XT_UNUSED(autoInc)) { } /* Add a contraint. If lastColumn is TRUE then add the contraint * to the last column. If not, expect addListedColumn() to be called. */ - virtual void addConstraint(XTThreadPtr self __attribute__((unused)), char *name __attribute__((unused)), u_int type __attribute__((unused)), bool lastColumn __attribute__((unused))) { + virtual void addConstraint(XTThreadPtr XT_UNUSED(self), char *XT_UNUSED(name), u_int XT_UNUSED(type), bool XT_UNUSED(lastColumn)) { } /* Move the last column created. If symbol is NULL then move the column to the * first position, else move it to the position just after the given column. */ - virtual void moveColumn(XTThreadPtr self __attribute__((unused)), char *col_name __attribute__((unused))) { + virtual void moveColumn(XTThreadPtr XT_UNUSED(self), char *XT_UNUSED(col_name)) { } - virtual void dropColumn(XTThreadPtr self __attribute__((unused)), char *col_name __attribute__((unused))) { + virtual void dropColumn(XTThreadPtr XT_UNUSED(self), char *XT_UNUSED(col_name)) { } - virtual void dropConstraint(XTThreadPtr self __attribute__((unused)), char *name __attribute__((unused)), u_int type __attribute__((unused))) { + virtual void dropConstraint(XTThreadPtr XT_UNUSED(self), char *XT_UNUSED(name), u_int XT_UNUSED(type)) { } - virtual void setIndexName(XTThreadPtr self __attribute__((unused)), char *name __attribute__((unused))) { + virtual void setIndexName(XTThreadPtr XT_UNUSED(self), char *XT_UNUSED(name)) { } - virtual void addListedColumn(XTThreadPtr self __attribute__((unused)), char *index_col_name __attribute__((unused))) { + virtual void addListedColumn(XTThreadPtr XT_UNUSED(self), char *XT_UNUSED(index_col_name)) { } - virtual void setReferencedTable(XTThreadPtr self __attribute__((unused)), char *ref_table __attribute__((unused))) { + virtual void setReferencedTable(XTThreadPtr XT_UNUSED(self), char *XT_UNUSED(ref_schema), char *XT_UNUSED(ref_table)) { } - virtual void addReferencedColumn(XTThreadPtr self __attribute__((unused)), char *index_col_name __attribute__((unused))) { + virtual void addReferencedColumn(XTThreadPtr XT_UNUSED(self), char *XT_UNUSED(index_col_name)) { } - virtual void setActions(XTThreadPtr self __attribute__((unused)), int on_delete __attribute__((unused)), int on_update __attribute__((unused))) { + virtual void setActions(XTThreadPtr XT_UNUSED(self), int XT_UNUSED(on_delete), int XT_UNUSED(on_update)) { } virtual void parseTable(XTThreadPtr self, bool convert, char *sql); @@ -859,7 +870,7 @@ void XTParseTable::parseAddTableItem(XTT if (pt_current->isKeyWord("CONSTRAINT")) { pt_current = pt_tokenizer->nextToken(self); if (pt_current->isIdentifier()) - parseQualifiedName(self, name); + parseQualifiedName(self, NULL, name); } if (pt_current->isReservedWord(XT_TK_PRIMARY)) { @@ -974,13 +985,15 @@ void XTParseTable::parseMoveColumn(XTThr char name[XT_IDENTIFIER_NAME_SIZE]; pt_current = pt_tokenizer->nextToken(self); - parseQualifiedName(self, name); + parseQualifiedName(self, NULL, name); moveColumn(self, name); } } -void XTParseTable::parseQualifiedName(XTThreadPtr self, char *name) +void XTParseTable::parseQualifiedName(XTThreadPtr self, char *parent_name, char *name) { + if (parent_name) + parent_name[0] = '\0'; /* Should be an identifier by I have this example: * CREATE TABLE t1 ( comment CHAR(32) ASCII NOT NULL, koi8_ru_f CHAR(32) CHARACTER SET koi8r NOT NULL default '' ) CHARSET=latin5; * @@ -990,6 +1003,8 @@ void XTParseTable::parseQualifiedName(XT raiseError(self, pt_current, XT_ERR_ID_TOO_LONG); pt_current = pt_tokenizer->nextToken(self); while (pt_current->isKeyWord(".")) { + if (parent_name) + xt_strcpy(XT_IDENTIFIER_NAME_SIZE,parent_name, name); pt_current = pt_tokenizer->nextToken(self); /* Accept anything after the DOT! */ if (pt_current->getString(name, XT_IDENTIFIER_NAME_SIZE) >= XT_IDENTIFIER_NAME_SIZE) @@ -1002,7 +1017,7 @@ void XTParseTable::parseTableName(XTThre { char name[XT_IDENTIFIER_NAME_SIZE]; - parseQualifiedName(self, name); + parseQualifiedName(self, NULL, name); setTableName(self, name, alterTable); } @@ -1011,7 +1026,7 @@ void XTParseTable::parseColumnDefinition char col_name[XT_IDENTIFIER_NAME_SIZE]; // column_definition - parseQualifiedName(self, col_name); + parseQualifiedName(self, NULL, col_name); addColumn(self, col_name, old_col_name); parseDataType(self); @@ -1111,7 +1126,7 @@ u_int XTParseTable::columnList(XTThreadP pt_current->expectKeyWord(self, "("); do { pt_current = pt_tokenizer->nextToken(self); - parseQualifiedName(self, name); + parseQualifiedName(self, NULL, name); addListedColumn(self, name); cols++; if (index_cols) { @@ -1135,19 +1150,20 @@ void XTParseTable::parseReferenceDefinit int on_delete = XT_KEY_ACTION_DEFAULT; int on_update = XT_KEY_ACTION_DEFAULT; char name[XT_IDENTIFIER_NAME_SIZE]; + char parent_name[XT_IDENTIFIER_NAME_SIZE]; u_int cols = 0; // REFERENCES tbl_name pt_current = pt_tokenizer->nextToken(self, "REFERENCES", pt_current); - parseQualifiedName(self, name); - setReferencedTable(self, name); + parseQualifiedName(self, parent_name, name); + setReferencedTable(self, parent_name[0] ? parent_name : NULL, name); // [ (index_col_name,...) ] if (pt_current->isKeyWord("(")) { pt_current->expectKeyWord(self, "("); do { pt_current = pt_tokenizer->nextToken(self); - parseQualifiedName(self, name); + parseQualifiedName(self, NULL, name); addReferencedColumn(self, name); cols++; if (cols > req_cols) @@ -1219,7 +1235,7 @@ void XTParseTable::parseAlterTable(XTThr if (pt_current->isReservedWord(XT_TK_COLUMN)) pt_current = pt_tokenizer->nextToken(self); - parseQualifiedName(self, old_col_name); + parseQualifiedName(self, NULL, old_col_name); parseColumnDefinition(self, old_col_name); parseMoveColumn(self); } @@ -1251,7 +1267,7 @@ void XTParseTable::parseAlterTable(XTThr else { if (pt_current->isReservedWord(XT_TK_COLUMN)) pt_current = pt_tokenizer->nextToken(self); - parseQualifiedName(self, name); + parseQualifiedName(self, NULL, name); dropColumn(self, name); } } @@ -1259,7 +1275,7 @@ void XTParseTable::parseAlterTable(XTThr pt_current = pt_tokenizer->nextToken(self); if (pt_current->isKeyWord("TO")) pt_current = pt_tokenizer->nextToken(self); - parseQualifiedName(self, name); + parseQualifiedName(self, NULL, name); } else /* Just ignore the syntax until the next , */ @@ -1284,7 +1300,7 @@ void XTParseTable::parseCreateIndex(XTTh else if (pt_current->isKeyWord("SPACIAL")) pt_current = pt_tokenizer->nextToken(self); pt_current = pt_tokenizer->nextToken(self, "INDEX", pt_current); - parseQualifiedName(self, name); + parseQualifiedName(self, NULL, name); optionalIndexType(self); pt_current = pt_tokenizer->nextToken(self, "ON", pt_current); parseTableName(self, true); @@ -1299,7 +1315,7 @@ void XTParseTable::parseDropIndex(XTThre pt_current = pt_tokenizer->nextToken(self, "DROP", pt_current); pt_current = pt_tokenizer->nextToken(self, "INDEX", pt_current); - parseQualifiedName(self, name); + parseQualifiedName(self, NULL, name); pt_current = pt_tokenizer->nextToken(self, "ON", pt_current); parseTableName(self, true); dropConstraint(self, name, XT_DD_INDEX); @@ -1340,7 +1356,7 @@ class XTCreateTable : public XTParseTabl virtual void addConstraint(XTThreadPtr self, char *name, u_int type, bool lastColumn); virtual void dropConstraint(XTThreadPtr self, char *name, u_int type); virtual void addListedColumn(XTThreadPtr self, char *index_col_name); - virtual void setReferencedTable(XTThreadPtr self, char *ref_table); + virtual void setReferencedTable(XTThreadPtr self, char *ref_schema, char *ref_table); virtual void addReferencedColumn(XTThreadPtr self, char *index_col_name); virtual void setActions(XTThreadPtr self, int on_delete, int on_update); @@ -1535,23 +1551,31 @@ void XTCreateTable::addListedColumn(XTTh } } -void XTCreateTable::setReferencedTable(XTThreadPtr self, char *ref_table) +void XTCreateTable::setReferencedTable(XTThreadPtr self, char *ref_schema, char *ref_table) { XTDDForeignKey *fk = (XTDDForeignKey *) ct_curr_constraint; char path[PATH_MAX]; - xt_strcpy(PATH_MAX, path, ct_tab_path->ps_path); - xt_remove_last_name_of_path(path); - if (ct_convert) { - char buffer[XT_IDENTIFIER_NAME_SIZE]; - size_t len; - - myxt_static_convert_identifier(self, ct_charset, ref_table, buffer, XT_IDENTIFIER_NAME_SIZE); - len = strlen(path); - myxt_static_convert_table_name(self, buffer, &path[len], PATH_MAX - len); - } - else + if (ref_schema) { + xt_strcpy(PATH_MAX,path, "."); + xt_add_dir_char(PATH_MAX, path); + xt_strcat(PATH_MAX, path, ref_schema); + xt_add_dir_char(PATH_MAX, path); xt_strcat(PATH_MAX, path, ref_table); + } else { + xt_strcpy(PATH_MAX, path, ct_tab_path->ps_path); + xt_remove_last_name_of_path(path); + if (ct_convert) { + char buffer[XT_IDENTIFIER_NAME_SIZE]; + size_t len; + + myxt_static_convert_identifier(self, ct_charset, ref_table, buffer, XT_IDENTIFIER_NAME_SIZE); + len = strlen(path); + myxt_static_convert_table_name(self, buffer, &path[len], PATH_MAX - len); + } + else + xt_strcat(PATH_MAX, path, ref_table); + } fk->fk_ref_tab_name = (XTPathStrPtr) xt_dup_string(self, path); } @@ -1578,7 +1602,7 @@ void XTCreateTable::addReferencedColumn( fk->fk_ref_cols.clone(self, &fk->co_cols); } -void XTCreateTable::setActions(XTThreadPtr self __attribute__((unused)), int on_delete, int on_update) +void XTCreateTable::setActions(XTThreadPtr XT_UNUSED(self), int on_delete, int on_update) { XTDDForeignKey *fk = (XTDDForeignKey *) ct_curr_constraint; @@ -1711,8 +1735,8 @@ void XTDDConstraint::alterColumnName(XTT void XTDDConstraint::getColumnList(char *buffer, size_t size) { if (co_table->dt_table) { - xt_strcat(size, buffer, "`"); - xt_strcpy(size, buffer, co_table->dt_table->tab_name->ps_path); + xt_strcpy(size, buffer, "`"); + xt_strcat(size, buffer, co_table->dt_table->tab_name->ps_path); xt_strcat(size, buffer, "` (`"); } else @@ -1739,6 +1763,20 @@ bool XTDDConstraint::sameColumns(XTDDCon return OK; } +bool XTDDConstraint::samePrefixColumns(XTDDConstraint *co) +{ + u_int i = 0; + + if (co_cols.size() > co->co_cols.size()) + return false; + while (i<co_cols.size()) { + if (myxt_strcasecmp(co_cols.itemAt(i)->cr_col_name, co->co_cols.itemAt(i)->cr_col_name) != 0) + return false; + i++; + } + return OK; +} + bool XTDDConstraint::attachColumns() { XTDDColumn *col; @@ -1773,6 +1811,7 @@ bool XTDDTableRef::checkReference(xtWord XTIdxSearchKeyRec search_key; xtXactID xn_id; XTXactWaitRec xw; + bool ok = false; if (!(loc_ind = tr_fkey->getReferenceIndexPtr())) return false; @@ -1792,40 +1831,42 @@ bool XTDDTableRef::checkReference(xtWord /* Search for the key in the child (referencing) table: */ if (!(ot = xt_db_open_table_using_tab(tr_fkey->co_table->dt_table, thread))) - goto failed; + return false; retry: if (!xt_idx_search(ot, ind, &search_key)) - goto failed; + goto done; while (ot->ot_curr_rec_id && search_key.sk_on_key) { switch (xt_tab_maybe_committed(ot, ot->ot_curr_rec_id, &xn_id, &ot->ot_curr_row_id, &ot->ot_curr_updated)) { case XT_MAYBE: xw.xw_xn_id = xn_id; if (!xt_xn_wait_for_xact(thread, &xw, NULL)) - goto failed; + goto done; goto retry; case XT_ERR: - goto failed; + goto done; case TRUE: /* We found a matching child: */ xt_register_ixterr(XT_REG_CONTEXT, XT_ERR_ROW_IS_REFERENCED, tr_fkey->co_name); - goto failed; - break; + goto done; case FALSE: if (!xt_idx_next(ot, ind, &search_key)) - goto failed; + goto done; break; } } /* No matching children, all OK: */ - xt_db_return_table_to_pool_ns(ot); - return true; + ok = true; - failed: + done: + if (ot->ot_ind_rhandle) { + xt_ind_release_handle(ot->ot_ind_rhandle, FALSE, thread); + ot->ot_ind_rhandle = NULL; + } xt_db_return_table_to_pool_ns(ot); - return false; + return ok; } /* @@ -1962,6 +2003,10 @@ bool XTDDTableRef::modifyRow(XTOpenTable } /* No matching children, all OK: */ + if (ot->ot_ind_rhandle) { + xt_ind_release_handle(ot->ot_ind_rhandle, FALSE, thread); + ot->ot_ind_rhandle = NULL; + } xt_db_return_table_to_pool_ns(ot); success: @@ -1971,6 +2016,10 @@ bool XTDDTableRef::modifyRow(XTOpenTable return true; failed_2: + if (ot->ot_ind_rhandle) { + xt_ind_release_handle(ot->ot_ind_rhandle, FALSE, thread); + ot->ot_ind_rhandle = NULL; + } xt_db_return_table_to_pool_ns(ot); failed: @@ -2055,8 +2104,13 @@ void XTDDForeignKey::finalize(XTThreadPt void XTDDForeignKey::loadString(XTThreadPtr self, XTStringBufferPtr sb) { + char schema_name[XT_IDENTIFIER_NAME_SIZE]; + XTDDConstraint::loadString(self, sb); xt_sb_concat(self, sb, " REFERENCES `"); + xt_2nd_last_name_of_path(XT_IDENTIFIER_NAME_SIZE, schema_name, fk_ref_tab_name->ps_path); + xt_sb_concat(self, sb, schema_name); + xt_sb_concat(self, sb, "`.`"); xt_sb_concat(self, sb, xt_last_name_of_path(fk_ref_tab_name->ps_path)); xt_sb_concat(self, sb, "` "); @@ -2136,6 +2190,20 @@ bool XTDDForeignKey::sameReferenceColumn return OK; } +bool XTDDForeignKey::samePrefixReferenceColumns(XTDDConstraint *co) +{ + u_int i = 0; + + if (fk_ref_cols.size() > co->co_cols.size()) + return false; + while (i<fk_ref_cols.size()) { + if (myxt_strcasecmp(fk_ref_cols.itemAt(i)->cr_col_name, co->co_cols.itemAt(i)->cr_col_name) != 0) + return false; + i++; + } + return OK; +} + bool XTDDForeignKey::checkReferencedTypes(XTDDTable *dt) { XTDDColumn *col, *ref_col; @@ -2288,6 +2356,10 @@ bool XTDDForeignKey::insertRow(xtWord1 * goto failed_2; case TRUE: /* We found a matching parent: */ + if (ot->ot_ind_rhandle) { + xt_ind_release_handle(ot->ot_ind_rhandle, FALSE, thread); + ot->ot_ind_rhandle = NULL; + } xt_db_return_table_to_pool_ns(ot); goto success; case FALSE: @@ -2300,6 +2372,10 @@ bool XTDDForeignKey::insertRow(xtWord1 * xt_register_ixterr(XT_REG_CONTEXT, XT_ERR_NO_REFERENCED_ROW, co_name); failed_2: + if (ot->ot_ind_rhandle) { + xt_ind_release_handle(ot->ot_ind_rhandle, FALSE, thread); + ot->ot_ind_rhandle = NULL; + } xt_db_return_table_to_pool_ns(ot); failed: @@ -2672,16 +2748,24 @@ void XTDDTable::checkForeignKeys(XTThrea XTDDIndex *XTDDTable::findIndex(XTDDConstraint *co) { - XTDDIndex *ind; + XTDDIndex *ind = NULL; + XTDDIndex *cur_ind; + u_int index_size = UINT_MAX; for (u_int i=0; i<dt_indexes.size(); i++) { - ind = dt_indexes.itemAt(i); - if (co->sameColumns(ind)) - return ind; + cur_ind = dt_indexes.itemAt(i); + u_int sz = cur_ind->getIndexPtr()->mi_key_size; + if (sz < index_size && co->samePrefixColumns(cur_ind)) { + ind = cur_ind; + index_size = sz; + } } + + if (ind) + return ind; + { char buffer[XT_ERR_MSG_SIZE - 200]; - co->getColumnList(buffer, XT_ERR_MSG_SIZE - 200); xt_register_ixterr(XT_REG_CONTEXT, XT_ERR_NO_MATCHING_INDEX, buffer); } @@ -2690,16 +2774,24 @@ XTDDIndex *XTDDTable::findIndex(XTDDCons XTDDIndex *XTDDTable::findReferenceIndex(XTDDForeignKey *fk) { - XTDDIndex *ind; + XTDDIndex *ind = NULL; + XTDDIndex *cur_ind; XTDDColumnRef *cr; u_int i; + u_int index_size = UINT_MAX; for (i=0; i<dt_indexes.size(); i++) { - ind = dt_indexes.itemAt(i); - if (fk->sameReferenceColumns(ind)) - return ind; + cur_ind = dt_indexes.itemAt(i); + u_int sz = cur_ind->getIndexPtr()->mi_key_size; + if (sz < index_size && fk->samePrefixReferenceColumns(cur_ind)) { + ind = cur_ind; + index_size = sz; + } } + if (ind) + return ind; + /* If the index does not exist, maybe the columns do not exist?! */ for (i=0; i<fk->fk_ref_cols.size(); i++) { cr = fk->fk_ref_cols.itemAt(i); @@ -2867,9 +2959,33 @@ bool XTDDTable::updateRow(XTOpenTablePtr return ok; } -xtBool XTDDTable::checkCanDrop() +/* + * drop_db parameter is TRUE if we are dropping the schema of this table. In this case + * we return TRUE if the table has only refs to the tables from its own schema + */ +xtBool XTDDTable::checkCanDrop(xtBool drop_db) { /* no refs or references only itself */ - return (dt_trefs == NULL) || - (dt_trefs->tr_next == NULL) && (dt_trefs->tr_fkey->co_table == this); + if ((dt_trefs == NULL) || ((dt_trefs->tr_next == NULL) && (dt_trefs->tr_fkey->co_table == this))) + return TRUE; + + if (!drop_db) + return FALSE; + + const char *this_schema = xt_last_2_names_of_path(dt_table->tab_name->ps_path); + size_t this_schema_sz = xt_last_name_of_path(dt_table->tab_name->ps_path) - this_schema; + XTDDTableRef *tr = dt_trefs; + + while (tr) { + const char *tab_path = tr->tr_fkey->co_table->dt_table->tab_name->ps_path; + const char *tab_schema = xt_last_2_names_of_path(tab_path); + size_t tab_schema_sz = xt_last_name_of_path(tab_path) - tab_schema; + + if (this_schema_sz != tab_schema_sz || strncmp(this_schema, tab_schema, tab_schema_sz)) + return FALSE; + + tr = tr->tr_next; + } + + return TRUE; } === modified file 'storage/pbxt/src/datadic_xt.h' --- a/storage/pbxt/src/datadic_xt.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/datadic_xt.h 2009-08-18 07:46:53 +0000 @@ -137,6 +137,7 @@ class XTDDColumnRef : public XTObject { return new_obj; } + virtual void init(XTThreadPtr self) { XTObject::init(self); } virtual void init(XTThreadPtr self, XTObject *obj); virtual void finalize(XTThreadPtr self); }; @@ -156,6 +157,7 @@ class XTDDConstraint : public XTObject { co_ind_name(NULL) { } + virtual void init(XTThreadPtr self) { XTObject::init(self); } virtual void init(XTThreadPtr self, XTObject *obj); virtual void finalize(XTThreadPtr self) { if (co_name) @@ -169,6 +171,7 @@ class XTDDConstraint : public XTObject { virtual void alterColumnName(XTThreadPtr self, char *from_name, char *to_name); void getColumnList(char *buffer, size_t size); bool sameColumns(XTDDConstraint *co); + bool samePrefixColumns(XTDDConstraint *co); bool attachColumns(); }; @@ -198,6 +201,7 @@ class XTDDIndex : public XTDDConstraint return new_obj; } + virtual void init(XTThreadPtr self) { XTDDConstraint::init(self); }; virtual void init(XTThreadPtr self, XTObject *obj); struct XTIndex *getIndexPtr(); }; @@ -230,12 +234,14 @@ class XTDDForeignKey : public XTDDIndex return new_obj; } + virtual void init(XTThreadPtr self) { XTDDIndex::init(self); } virtual void init(XTThreadPtr self, XTObject *obj); virtual void finalize(XTThreadPtr self); virtual void loadString(XTThreadPtr self, XTStringBufferPtr sb); void getReferenceList(char *buffer, size_t size); struct XTIndex *getReferenceIndexPtr(); bool sameReferenceColumns(XTDDConstraint *co); + bool samePrefixReferenceColumns(XTDDConstraint *co); bool checkReferencedTypes(XTDDTable *dt); void removeReference(XTThreadPtr self); bool insertRow(xtWord1 *before, xtWord1 *after, XTThreadPtr thread); @@ -284,7 +290,7 @@ class XTDDTable : public XTObject { XTDDIndex *findReferenceIndex(XTDDForeignKey *fk); bool insertRow(struct XTOpenTable *rec_ot, xtWord1 *buffer); bool checkNoAction(struct XTOpenTable *ot, xtRecordID rec_id); - xtBool checkCanDrop(); + xtBool checkCanDrop(xtBool drop_db); bool deleteRow(struct XTOpenTable *rec_ot, xtWord1 *buffer); void deleteAllRows(XTThreadPtr self); bool updateRow(struct XTOpenTable *rec_ot, xtWord1 *before, xtWord1 *after); === modified file 'storage/pbxt/src/datalog_xt.cc' --- a/storage/pbxt/src/datalog_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/datalog_xt.cc 2009-08-18 07:46:53 +0000 @@ -69,6 +69,7 @@ xtBool XTDataSeqRead::sl_seq_init(struct sl_rec_log_id = 0; sl_rec_log_offset = 0; sl_record_len = 0; + sl_extra_garbage = 0; return sl_buffer != NULL; } @@ -130,8 +131,25 @@ xtBool XTDataSeqRead::sl_rnd_read(xtLogO /* * Unlike the transaction log sequential reader, this function only returns * the header of a record. + * + * {SKIP-GAPS} + * This function now skips gaps. This should not be required, because in normal + * operation, no gaps should be created. + * + * However, if his happens there is a danger that a valid record after the + * gap will be lost. + * + * So, if we find an invalid record, we scan through the log to find the next + * valid record. Note, that there is still a danger that will will find + * data that looks like a valid record, but is not. + * + * In this case, this "pseudo record" may cause the function to actually skip + * valid records. + * + * Note, any such malfunction will eventually cause the record to be lost forever + * after the garbage collector has run. */ -xtBool XTDataSeqRead::sl_seq_next(XTXactLogBufferDPtr *ret_entry, xtBool verify, struct XTThread *thread) +xtBool XTDataSeqRead::sl_seq_next(XTXactLogBufferDPtr *ret_entry, struct XTThread *thread) { XTXactLogBufferDPtr record; size_t tfer; @@ -140,10 +158,12 @@ xtBool XTDataSeqRead::sl_seq_next(XTXact size_t max_rec_len; xtBool reread_from_buffer; xtWord4 size; + xtLogOffset gap_start = 0; /* Go to the next record (xseq_record_len must be initialized * to 0 for this to work. */ + retry: sl_rec_log_offset += sl_record_len; sl_record_len = 0; @@ -174,6 +194,8 @@ xtBool XTDataSeqRead::sl_seq_next(XTXact record = (XTXactLogBufferDPtr) (sl_buffer + rec_offset); switch (record->xl.xl_status_1) { case XT_LOG_ENT_HEADER: + if (sl_rec_log_offset != 0) + goto scan_to_next_record; if (offsetof(XTXactLogHeaderDRec, xh_size_4) + 4 > max_rec_len) { reread_from_buffer = TRUE; goto read_more; @@ -183,33 +205,42 @@ xtBool XTDataSeqRead::sl_seq_next(XTXact reread_from_buffer = TRUE; goto read_more; } - if (verify) { - if (record->xh.xh_checksum_1 != XT_CHECKSUM_1(sl_rec_log_id)) - goto return_empty; - if (XT_LOG_HEAD_MAGIC(record, len) != XT_LOG_FILE_MAGIC) + + if (record->xh.xh_checksum_1 != XT_CHECKSUM_1(sl_rec_log_id)) + goto return_empty; + if (XT_LOG_HEAD_MAGIC(record, len) != XT_LOG_FILE_MAGIC) + goto return_empty; + if (len > offsetof(XTXactLogHeaderDRec, xh_log_id_4) + 4) { + if (XT_GET_DISK_4(record->xh.xh_log_id_4) != sl_rec_log_id) goto return_empty; - if (len > offsetof(XTXactLogHeaderDRec, xh_log_id_4) + 4) { - if (XT_GET_DISK_4(record->xh.xh_log_id_4) != sl_rec_log_id) - goto return_empty; - } } break; case XT_LOG_ENT_EXT_REC_OK: case XT_LOG_ENT_EXT_REC_DEL: + if (gap_start) { + xt_logf(XT_NS_CONTEXT, XT_LOG_WARNING, "Gap in data log %lu, start: %llu, size: %llu\n", (u_long) sl_rec_log_id, (u_llong) gap_start, (u_llong) (sl_rec_log_offset - gap_start)); + gap_start = 0; + } len = offsetof(XTactExtRecEntryDRec, er_data); if (len > max_rec_len) { reread_from_buffer = TRUE; goto read_more; } size = XT_GET_DISK_4(record->er.er_data_size_4); - if (verify) { - if (sl_rec_log_offset + (xtLogOffset) offsetof(XTactExtRecEntryDRec, er_data) + size > sl_log_eof) - goto return_empty; - } + /* Verify the record as good as we can! */ + if (!size) + goto scan_to_next_record; + if (sl_rec_log_offset + (xtLogOffset) offsetof(XTactExtRecEntryDRec, er_data) + size > sl_log_eof) + goto scan_to_next_record; + if (!XT_GET_DISK_4(record->er.er_tab_id_4)) + goto scan_to_next_record; + if (!XT_GET_DISK_4(record->er.er_rec_id_4)) + goto scan_to_next_record; break; default: - ASSERT_NS(FALSE); - goto return_empty; + /* Note, we no longer assume EOF. + * Instead, we skip to the next value record. */ + goto scan_to_next_record; } if (len <= max_rec_len) { @@ -243,7 +274,20 @@ xtBool XTDataSeqRead::sl_seq_next(XTXact *ret_entry = (XTXactLogBufferDPtr) sl_buffer; return OK; + scan_to_next_record: + if (!gap_start) { + gap_start = sl_rec_log_offset; + xt_logf(XT_NS_CONTEXT, XT_LOG_WARNING, "Gap found in data log %lu, starting at offset %llu\n", (u_long) sl_rec_log_id, (u_llong) gap_start); + } + sl_record_len = 1; + sl_extra_garbage++; + goto retry; + return_empty: + if (gap_start) { + xt_logf(XT_NS_CONTEXT, XT_LOG_WARNING, "Gap in data log %lu, start: %llu, size: %llu\n", (u_long) sl_rec_log_id, (u_llong) gap_start, (u_llong) (sl_rec_log_offset - gap_start)); + gap_start = 0; + } *ret_entry = NULL; return OK; } @@ -285,22 +329,54 @@ static xtBool dl_create_log_header(XTDat return OK; } -static xtBool dl_write_log_header(XTDataLogFilePtr data_log, XTOpenFilePtr of, xtBool flush, XTThreadPtr thread) +static xtBool dl_write_garbage_level(XTDataLogFilePtr data_log, XTOpenFilePtr of, xtBool flush, XTThreadPtr thread) { XTXactLogHeaderDRec header; /* The header was not completely written, so write a new one: */ XT_SET_DISK_8(header.xh_free_space_8, data_log->dlf_garbage_count); - XT_SET_DISK_8(header.xh_file_len_8, data_log->dlf_log_eof); - XT_SET_DISK_8(header.xh_comp_pos_8, data_log->dlf_start_offset); - - if (!xt_pwrite_file(of, offsetof(XTXactLogHeaderDRec, xh_free_space_8), 24, (xtWord1 *) &header.xh_free_space_8, &thread->st_statistics.st_data, thread)) + if (!xt_pwrite_file(of, offsetof(XTXactLogHeaderDRec, xh_free_space_8), 8, (xtWord1 *) &header.xh_free_space_8, &thread->st_statistics.st_data, thread)) return FAILED; if (flush && !xt_flush_file(of, &thread->st_statistics.st_data, thread)) return FAILED; return OK; } +/* + * {SKIP-GAPS} + * Extra garbage is the amount of space skipped during recovery of the data + * log file. We assume this space has not be counted as garbage, + * and add it to the garbage count. + * + * This may mean that our estimate of garbaged is higher than it should + * be, but that is better than the other way around. + * + * The fact is, there should not be any gaps in the data log files, so + * this is actually an exeption which should not occur. + */ +static xtBool dl_write_log_header(XTDataLogFilePtr data_log, XTOpenFilePtr of, xtLogOffset extra_garbage, XTThreadPtr thread) +{ + XTXactLogHeaderDRec header; + + XT_SET_DISK_8(header.xh_file_len_8, data_log->dlf_log_eof); + + if (extra_garbage) { + data_log->dlf_garbage_count += extra_garbage; + if (data_log->dlf_garbage_count > data_log->dlf_log_eof) + data_log->dlf_garbage_count = data_log->dlf_log_eof; + XT_SET_DISK_8(header.xh_free_space_8, data_log->dlf_garbage_count); + if (!xt_pwrite_file(of, offsetof(XTXactLogHeaderDRec, xh_free_space_8), 16, (xtWord1 *) &header.xh_free_space_8, &thread->st_statistics.st_data, thread)) + return FAILED; + } + else { + if (!xt_pwrite_file(of, offsetof(XTXactLogHeaderDRec, xh_file_len_8), 8, (xtWord1 *) &header.xh_file_len_8, &thread->st_statistics.st_data, thread)) + return FAILED; + } + if (!xt_flush_file(of, &thread->st_statistics.st_data, thread)) + return FAILED; + return OK; +} + static void dl_free_seq_read(XTThreadPtr self __attribute__((unused)), XTDataSeqReadPtr seq_read) { seq_read->sl_seq_exit(); @@ -318,7 +394,7 @@ static void dl_recover_log(XTThreadPtr s seq_read.sl_seq_start(data_log->dlf_log_id, 0, FALSE); for (;;) { - if (!seq_read.sl_seq_next(&record, TRUE, self)) + if (!seq_read.sl_seq_next(&record, self)) xt_throw(self); if (!record) break; @@ -331,13 +407,18 @@ static void dl_recover_log(XTThreadPtr s } } - if (!(data_log->dlf_log_eof = seq_read.sl_rec_log_offset)) { + ASSERT_NS(seq_read.sl_log_eof == seq_read.sl_rec_log_offset); + data_log->dlf_log_eof = seq_read.sl_rec_log_offset; + + if (data_log->dlf_log_eof < sizeof(XTXactLogHeaderDRec)) { data_log->dlf_log_eof = sizeof(XTXactLogHeaderDRec); if (!dl_create_log_header(data_log, seq_read.sl_log_file, self)) xt_throw(self); } - if (!dl_write_log_header(data_log, seq_read.sl_log_file, TRUE, self)) - xt_throw(self); + else { + if (!dl_write_log_header(data_log, seq_read.sl_log_file, seq_read.sl_extra_garbage, self)) + xt_throw(self); + } freer_(); // dl_free_seq_read(&seq_read) } @@ -452,7 +533,7 @@ xtBool XTDataLogCache::dls_set_log_state return FAILED; } -static int dl_cmp_log_id(XTThreadPtr XT_UNUSED(self), register const void XT_UNUSED(*thunk), register const void *a, register const void *b) +static int dl_cmp_log_id(XTThreadPtr XT_UNUSED(self), register const void *XT_UNUSED(thunk), register const void *a, register const void *b) { xtLogID log_id_a = *((xtLogID *) a); xtLogID log_id_b = *((xtLogID *) b); @@ -1110,7 +1191,6 @@ xtBool XTDataLogBuffer::dlb_get_log_offs *log_id = dlb_data_log->dlf_log_id; *out_offset = dlb_data_log->dlf_log_eof; - dlb_data_log->dlf_log_eof += req_size; return OK; } @@ -1149,7 +1229,7 @@ xtBool XTDataLogBuffer::dlb_flush_log(xt return OK; } -xtBool XTDataLogBuffer::dlb_write_thru_log(xtLogID log_id __attribute__((unused)), xtLogOffset log_offset, size_t size, xtWord1 *data, XTThreadPtr thread) +xtBool XTDataLogBuffer::dlb_write_thru_log(xtLogID XT_NDEBUG_UNUSED(log_id), xtLogOffset log_offset, size_t size, xtWord1 *data, XTThreadPtr thread) { ASSERT_NS(log_id == dlb_data_log->dlf_log_id); @@ -1158,6 +1238,11 @@ xtBool XTDataLogBuffer::dlb_write_thru_l if (!xt_pwrite_file(dlb_data_log->dlf_log_file, log_offset, size, data, &thread->st_statistics.st_data, thread)) return FAILED; + /* Increment of dlb_data_log->dlf_log_eof was moved here from dlb_get_log_offset() + * to ensure it is done after a successful update of the log, otherwise otherwise a + * gap occurs in the log which cause eof to be detected in middle of the log + */ + dlb_data_log->dlf_log_eof += size; #ifdef DEBUG if (log_offset + size > dlb_max_write_offset) dlb_max_write_offset = log_offset + size; @@ -1166,7 +1251,7 @@ xtBool XTDataLogBuffer::dlb_write_thru_l return OK; } -xtBool XTDataLogBuffer::dlb_append_log(xtLogID log_id __attribute__((unused)), xtLogOffset log_offset, size_t size, xtWord1 *data, XTThreadPtr thread) +xtBool XTDataLogBuffer::dlb_append_log(xtLogID XT_NDEBUG_UNUSED(log_id), xtLogOffset log_offset, size_t size, xtWord1 *data, XTThreadPtr thread) { ASSERT_NS(log_id == dlb_data_log->dlf_log_id); @@ -1179,10 +1264,12 @@ xtBool XTDataLogBuffer::dlb_append_log(x if (dlb_buffer_size >= dlb_buffer_len + size) { memcpy(dlb_log_buffer + dlb_buffer_len, data, size); dlb_buffer_len += size; + dlb_data_log->dlf_log_eof += size; return OK; } } - dlb_flush_log(FALSE, thread); + if (dlb_flush_log(FALSE, thread) != OK) + return FAILED; } ASSERT_NS(dlb_buffer_len == 0); @@ -1191,6 +1278,7 @@ xtBool XTDataLogBuffer::dlb_append_log(x dlb_buffer_offset = log_offset; dlb_buffer_len = size; memcpy(dlb_log_buffer, data, size); + dlb_data_log->dlf_log_eof += size; return OK; } @@ -1202,6 +1290,7 @@ xtBool XTDataLogBuffer::dlb_append_log(x dlb_max_write_offset = log_offset + size; #endif dlb_flush_required = TRUE; + dlb_data_log->dlf_log_eof += size; return OK; } @@ -1306,7 +1395,7 @@ xtBool XTDataLogBuffer::dlb_delete_log(x xt_lock_mutex_ns(&dlb_db->db_datalogs.dlc_head_lock); dlb_data_log->dlf_garbage_count += offsetof(XTactExtRecEntryDRec, er_data) + size; ASSERT_NS(dlb_data_log->dlf_garbage_count < dlb_data_log->dlf_log_eof); - if (!dl_write_log_header(dlb_data_log, dlb_data_log->dlf_log_file, FALSE, thread)) { + if (!dl_write_garbage_level(dlb_data_log, dlb_data_log->dlf_log_file, FALSE, thread)) { xt_unlock_mutex_ns(&dlb_db->db_datalogs.dlc_head_lock); return FAILED; } @@ -1329,7 +1418,7 @@ xtBool XTDataLogBuffer::dlb_delete_log(x xt_lock_mutex_ns(&dlb_db->db_datalogs.dlc_head_lock); data_log->dlf_garbage_count += offsetof(XTactExtRecEntryDRec, er_data) + size; ASSERT_NS(data_log->dlf_garbage_count < data_log->dlf_log_eof); - if (!dl_write_log_header(data_log, open_log->odl_log_file, FALSE, thread)) { + if (!dl_write_garbage_level(data_log, open_log->odl_log_file, FALSE, thread)) { xt_unlock_mutex_ns(&dlb_db->db_datalogs.dlc_head_lock); goto failed; } @@ -1357,7 +1446,7 @@ xtBool XTDataLogBuffer::dlb_delete_log(x * Delete all the extended data belonging to a particular * table. */ -xtPublic void xt_dl_delete_ext_data(XTThreadPtr self, XTTableHPtr tab, xtBool missing_ok __attribute__((unused)), xtBool have_table_lock) +xtPublic void xt_dl_delete_ext_data(XTThreadPtr self, XTTableHPtr tab, xtBool XT_UNUSED(missing_ok), xtBool have_table_lock) { XTOpenTablePtr ot; xtRecordID page_rec_id, offs_rec_id; @@ -1674,7 +1763,7 @@ static xtBool dl_collect_garbage(XTThrea xt_lock_mutex_ns(&db->db_datalogs.dlc_head_lock); data_log->dlf_garbage_count += garbage_count; ASSERT(data_log->dlf_garbage_count < data_log->dlf_log_eof); - if (!dl_write_log_header(data_log, cs.cs_seqread->sl_seq_open_file(), TRUE, self)) { + if (!dl_write_garbage_level(data_log, cs.cs_seqread->sl_seq_open_file(), TRUE, self)) { xt_unlock_mutex_ns(&db->db_datalogs.dlc_head_lock); xt_throw(self); } @@ -1683,7 +1772,7 @@ static xtBool dl_collect_garbage(XTThrea freer_(); // dl_free_compactor_state(&cs) return FAILED; } - if (!cs.cs_seqread->sl_seq_next(&record, TRUE, self)) + if (!cs.cs_seqread->sl_seq_next(&record, self)) xt_throw(self); cs.cs_seqread->sl_seq_pos(&curr_log_id, &curr_log_offset); if (!record) { @@ -1809,7 +1898,7 @@ static xtBool dl_collect_garbage(XTThrea xt_lock_mutex_ns(&db->db_datalogs.dlc_head_lock); data_log->dlf_garbage_count += garbage_count; ASSERT(data_log->dlf_garbage_count < data_log->dlf_log_eof); - if (!dl_write_log_header(data_log, cs.cs_seqread->sl_seq_open_file(), TRUE, self)) { + if (!dl_write_garbage_level(data_log, cs.cs_seqread->sl_seq_open_file(), TRUE, self)) { xt_unlock_mutex_ns(&db->db_datalogs.dlc_head_lock); xt_throw(self); } === modified file 'storage/pbxt/src/datalog_xt.h' --- a/storage/pbxt/src/datalog_xt.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/datalog_xt.h 2009-08-18 07:46:53 +0000 @@ -183,8 +183,8 @@ typedef struct XTSeqLogRead { virtual xtBool sl_rnd_read(xtLogOffset log_offset, size_t size, xtWord1 *data, size_t *read, struct XTThread *thread) { (void) log_offset; (void) size; (void) data; (void) read; (void) thread; return OK; }; - virtual xtBool sl_seq_next(XTXactLogBufferDPtr *entry, xtBool verify, struct XTThread *thread) { - (void) entry; (void) verify; (void) thread; return OK; + virtual xtBool sl_seq_next(XTXactLogBufferDPtr *entry, struct XTThread *thread) { + (void) entry; (void) thread; return OK; }; virtual void sl_seq_skip(size_t size) { (void) size; } } XTSeqLogReadRec, *XTSeqLogReadPtr; @@ -195,6 +195,7 @@ typedef struct XTDataSeqRead : public XT xtLogOffset sl_rec_log_offset; /* The current log read position. */ size_t sl_record_len; /* The length of the current record. */ xtLogOffset sl_log_eof; + xtLogOffset sl_extra_garbage; /* Garbage found during a scan. */ size_t sl_buffer_size; /* Size of the buffer. */ xtLogOffset sl_buf_log_offset; /* File offset of the buffer. */ @@ -208,7 +209,7 @@ typedef struct XTDataSeqRead : public XT virtual void sl_seq_pos(xtLogID *log_id, xtLogOffset *log_offset); virtual xtBool sl_seq_start(xtLogID log_id, xtLogOffset log_offset, xtBool missing_ok); virtual xtBool sl_rnd_read(xtLogOffset log_offset, size_t size, xtWord1 *data, size_t *read, struct XTThread *thread); - virtual xtBool sl_seq_next(XTXactLogBufferDPtr *entry, xtBool verify, struct XTThread *thread); + virtual xtBool sl_seq_next(XTXactLogBufferDPtr *entry, struct XTThread *thread); virtual void sl_seq_skip(size_t size); virtual void sl_seq_skip_to(off_t offset); } XTDataSeqReadRec, *XTDataSeqReadPtr; === modified file 'storage/pbxt/src/discover_xt.cc' --- a/storage/pbxt/src/discover_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/discover_xt.cc 2009-08-17 11:12:36 +0000 @@ -1282,9 +1282,11 @@ warn: #endif // LOCK_OPEN_HACK_REQUIRED //------------------------------ -int xt_create_table_frm(handlerton *hton, THD* thd, const char *db, const char *name, DT_FIELD_INFO *info, DT_KEY_INFO *keys __attribute__((unused)), xtBool skip_existing) +int xt_create_table_frm(handlerton *hton, THD* thd, const char *db, const char *name, DT_FIELD_INFO *info, DT_KEY_INFO *XT_UNUSED(keys), xtBool skip_existing) { #ifdef DRIZZLED + drizzled::message::Table table_proto; + static const char *ext = ".dfe"; static const int ext_len = 4; #else @@ -1329,8 +1331,7 @@ int xt_create_table_frm(handlerton *hton info->field_flags, COLUMN_FORMAT_TYPE_FIXED, NULL /*default_value*/, NULL /*on_update_value*/, &comment, NULL /*change*/, - NULL /*interval_list*/, info->field_charset, - NULL /*vcol_info*/)) + NULL /*interval_list*/, info->field_charset)) #else if (add_field_to_list(thd, &field_name, info->field_type, field_length_ptr, info->field_decimal_length, info->field_flags, @@ -1365,7 +1366,10 @@ int xt_create_table_frm(handlerton *hton /* Create an internal temp table */ #ifdef DRIZZLED - if (mysql_create_table_no_lock(thd, db, name, &mylex.create_info, &mylex.alter_info, 1, 0, false)) + table_proto.set_name(name); + table_proto.set_type(drizzled::message::Table::STANDARD); + + if (mysql_create_table_no_lock(thd, db, name, &mylex.create_info, &table_proto, &mylex.alter_info, 1, 0, false)) goto error; #else if (mysql_create_table_no_lock(thd, db, name, &mylex.create_info, &mylex.alter_info, 1, 0)) === modified file 'storage/pbxt/src/filesys_xt.cc' --- a/storage/pbxt/src/filesys_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/filesys_xt.cc 2009-08-18 07:46:53 +0000 @@ -23,6 +23,10 @@ #include "xt_config.h" +#ifdef DRIZZLED +#include <bitset> +#endif + #ifndef XT_WIN #include <unistd.h> #include <dirent.h> @@ -50,6 +54,7 @@ //#define DEBUG_TRACE_IO //#define DEBUG_TRACE_MAP_IO //#define DEBUG_TRACE_FILES +//#define INJECT_WRITE_REMAP_ERROR #endif #ifdef DEBUG_TRACE_FILES @@ -57,6 +62,11 @@ #define PRINTF xt_trace #endif +#ifdef INJECT_WRITE_REMAP_ERROR +#define INJECT_REMAP_FILE_SIZE 1000000 +#define INJECT_REMAP_FILE_TYPE "xtd" +#endif + /* ---------------------------------------------------------------------- * Globals */ @@ -127,11 +137,11 @@ static void fs_close_fmap(XTThreadPtr se mm->mm_start = NULL; } #endif - xt_rwmutex_free(self, &mm->mm_lock); + FILE_MAP_FREE_LOCK(self, &mm->mm_lock); xt_free(self, mm); } -static void fs_free_file(XTThreadPtr self, void *thunk __attribute__((unused)), void *item) +static void fs_free_file(XTThreadPtr self, void *XT_UNUSED(thunk), void *item) { XTFilePtr file_ptr = *((XTFilePtr *) item); @@ -148,17 +158,13 @@ static void fs_free_file(XTThreadPtr sel file_ptr->fil_filedes = XT_NULL_FD; } - if (file_ptr->fil_memmap) { - fs_close_fmap(self, file_ptr->fil_memmap); - file_ptr->fil_memmap = NULL; - } - #ifdef DEBUG_TRACE_FILES PRINTF("%s: free file: (%d) %s\n", self->t_name, (int) file_ptr->fil_id, file_ptr->fil_path ? xt_last_2_names_of_path(file_ptr->fil_path) : "?"); #endif if (!file_ptr->fil_ref_count) { + ASSERT_NS(!file_ptr->fil_handle_count); /* Flush any cache before this file is invalid: */ if (file_ptr->fil_path) { xt_free(self, file_ptr->fil_path); @@ -169,7 +175,7 @@ static void fs_free_file(XTThreadPtr sel } } -static int fs_comp_file(XTThreadPtr self __attribute__((unused)), register const void *thunk __attribute__((unused)), register const void *a, register const void *b) +static int fs_comp_file(XTThreadPtr XT_UNUSED(self), register const void *XT_UNUSED(thunk), register const void *a, register const void *b) { char *file_name = (char *) a; XTFilePtr file_ptr = *((XTFilePtr *) b); @@ -177,7 +183,7 @@ static int fs_comp_file(XTThreadPtr self return strcmp(file_name, file_ptr->fil_path); } -static int fs_comp_file_ci(XTThreadPtr self __attribute__((unused)), register const void *thunk __attribute__((unused)), register const void *a, register const void *b) +static int fs_comp_file_ci(XTThreadPtr XT_UNUSED(self), register const void *XT_UNUSED(thunk), register const void *a, register const void *b) { char *file_name = (char *) a; XTFilePtr file_ptr = *((XTFilePtr *) b); @@ -868,11 +874,22 @@ xtPublic xtBool xt_flush_file(XTOpenFile goto failed; } #else + /* Mac OS X has problems with fsync. We had several cases of index corruption presumably because + * fsync didn't really flush index pages to disk. fcntl(F_FULLFSYNC) is considered more effective + * in such case. + */ +#ifdef F_FULLFSYNC + if (fcntl(of->of_filedes, F_FULLFSYNC, 0) == -1) { + xt_register_ferrno(XT_REG_CONTEXT, errno, xt_file_path(of)); + goto failed; + } +#else if (fsync(of->of_filedes) == -1) { xt_register_ferrno(XT_REG_CONTEXT, errno, xt_file_path(of)); goto failed; } #endif +#endif #ifdef DEBUG_TRACE_IO xt_trace("/* %s */ pbxt_file_sync(\"%s\");\n", xt_trace_clock_diff(timef, start), of->fr_file->fil_path); #endif @@ -938,6 +955,29 @@ xtBool xt_pread_file(XTOpenFilePtr of, o return OK; } +xtPublic xtBool xt_lock_file_ptr(XTOpenFilePtr of, xtWord1 **data, off_t offset, size_t size, XTIOStatsPtr stat, XTThreadPtr thread) +{ + size_t red_size; + + if (!*data) { + if (!(*data = (xtWord1 *) xt_malloc_ns(size))) + return FAILED; + } + + if (!xt_pread_file(of, offset, size, 0, *data, &red_size, stat, thread)) + return FAILED; + + //if (red_size < size) + // memset(); + return OK; +} + +xtPublic void xt_unlock_file_ptr(XTOpenFilePtr XT_UNUSED(of), xtWord1 *data, XTThreadPtr XT_UNUSED(thread)) +{ + if (data) + xt_free_ns(data); +} + /* ---------------------------------------------------------------------- * Directory operations */ @@ -949,7 +989,13 @@ XTOpenDirPtr xt_dir_open(XTThreadPtr sel { XTOpenDirPtr od; - pushsr_(od, xt_dir_close, (XTOpenDirPtr) xt_calloc(self, sizeof(XTOpenDirRec))); +#ifdef XT_SOLARIS + /* see the comment in filesys_xt.h */ + size_t sz = pathconf(path, _PC_NAME_MAX) + sizeof(XTOpenDirRec) + 1; +#else + size_t sz = sizeof(XTOpenDirRec); +#endif + pushsr_(od, xt_dir_close, (XTOpenDirPtr) xt_calloc(self, sz)); #ifdef XT_WIN size_t len; @@ -976,7 +1022,6 @@ XTOpenDirPtr xt_dir_open(XTThreadPtr sel if (!od->od_dir) xt_throw_ferrno(XT_CONTEXT, errno, path); #endif - popr_(); // Discard xt_dir_close(od) return od; } @@ -1097,7 +1142,7 @@ xtBool xt_dir_next(XTThreadPtr self, XTO } #endif -char *xt_dir_name(XTThreadPtr self __attribute__((unused)), XTOpenDirPtr od) +char *xt_dir_name(XTThreadPtr XT_UNUSED(self), XTOpenDirPtr od) { #ifdef XT_WIN return od->od_data.cFileName; @@ -1106,8 +1151,9 @@ char *xt_dir_name(XTThreadPtr self __att #endif } -xtBool xt_dir_is_file(XTThreadPtr self __attribute__((unused)), XTOpenDirPtr od) +xtBool xt_dir_is_file(XTThreadPtr self, XTOpenDirPtr od) { + (void) self; #ifdef XT_WIN if (od->od_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) return FALSE; @@ -1156,6 +1202,15 @@ off_t xt_dir_file_size(XTThreadPtr self, static xtBool fs_map_file(XTFileMemMapPtr mm, XTFilePtr file, xtBool grow) { +#ifdef INJECT_WRITE_REMAP_ERROR + if (xt_is_extension(file->fil_path, INJECT_REMAP_FILE_TYPE)) { + if (mm->mm_length > INJECT_REMAP_FILE_SIZE) { + xt_register_ferrno(XT_REG_CONTEXT, 30, file->fil_path); + return FAILED; + } + } +#endif + ASSERT_NS(!mm->mm_start); #ifdef XT_WIN /* This will grow the file to the given size: */ @@ -1228,7 +1283,7 @@ xtPublic XTMapFilePtr xt_open_fmap(XTThr /* NULL is the value returned on error! */ mm->mm_mapdes = NULL; #endif - xt_rwmutex_init_with_autoname(self, &mm->mm_lock); + FILE_MAP_INIT_LOCK(self, &mm->mm_lock); mm->mm_length = fs_seek_eof(self, map->fr_file->fil_filedes, map->fr_file); if (sizeof(size_t) == 4 && mm->mm_length >= (off_t) 0xFFFFFFFF) xt_throw_ixterr(XT_CONTEXT, XT_ERR_FILE_TOO_LONG, map->fr_file->fil_path); @@ -1257,21 +1312,19 @@ xtPublic XTMapFilePtr xt_open_fmap(XTThr xtPublic void xt_close_fmap(XTThreadPtr self, XTMapFilePtr map) { + ASSERT_NS(!map->mf_slock_count); if (map->fr_file) { - xt_fs_release_file(self, map->fr_file); - xt_sl_lock(self, fs_globals.fsg_open_files); - pushr_(xt_sl_unlock, fs_globals.fsg_open_files); - + pushr_(xt_sl_unlock, fs_globals.fsg_open_files); map->fr_file->fil_handle_count--; - if (!map->fr_file->fil_handle_count) - fs_free_file(self, NULL, &map->fr_file); - + if (!map->fr_file->fil_handle_count) { + fs_close_fmap(self, map->fr_file->fil_memmap); + map->fr_file->fil_memmap = NULL; + } freer_(); - + + xt_fs_release_file(self, map->fr_file); map->fr_file = NULL; - - } map->mf_memmap = NULL; xt_free(self, map); @@ -1346,14 +1399,23 @@ static xtBool fs_remap_file(XTMapFilePtr } mm->mm_start = NULL; #ifdef XT_WIN - if (!CloseHandle(mm->mm_mapdes)) + /* It is possible that a previous remap attempt has failed: the map was closed + * but the new map was not allocated (e.g. because of insufficient disk space). + * In this case mm->mm_mapdes will be NULL. + */ + if (mm->mm_mapdes && !CloseHandle(mm->mm_mapdes)) return xt_register_ferrno(XT_REG_CONTEXT, fs_get_win_error(), xt_file_path(map)); mm->mm_mapdes = NULL; #endif + off_t old_size = mm->mm_length; mm->mm_length = new_size; - if (!fs_map_file(mm, map->fr_file, TRUE)) + if (!fs_map_file(mm, map->fr_file, TRUE)) { + /* Try to restore old mapping */ + mm->mm_length = old_size; + fs_map_file(mm, map->fr_file, FALSE); return FAILED; + } } return OK; @@ -1367,16 +1429,19 @@ static xtBool fs_remap_file(XTMapFilePtr xtPublic xtBool xt_pwrite_fmap(XTMapFilePtr map, off_t offset, size_t size, void *data, XTIOStatsPtr stat, XTThreadPtr thread) { XTFileMemMapPtr mm = map->mf_memmap; +#ifndef FILE_MAP_USE_PTHREAD_RW xtThreadID thd_id = thread->t_id; +#endif #ifdef DEBUG_TRACE_MAP_IO xt_trace("/* %s */ pbxt_fmap_writ(\"%s\", %lu, %lu);\n", xt_trace_clock_diff(NULL), map->fr_file->fil_path, (u_long) offset, (u_long) size); #endif - xt_rwmutex_slock(&mm->mm_lock, thd_id); + ASSERT_NS(!map->mf_slock_count); + FILE_MAP_READ_LOCK(&mm->mm_lock, thd_id); if (!mm->mm_start || offset + (off_t) size > mm->mm_length) { - xt_rwmutex_unlock(&mm->mm_lock, thd_id); + FILE_MAP_UNLOCK(&mm->mm_lock, thd_id); - xt_rwmutex_xlock(&mm->mm_lock, thd_id); + FILE_MAP_WRITE_LOCK(&mm->mm_lock, thd_id); if (!fs_remap_file(map, offset, size, stat)) goto failed; } @@ -1396,29 +1461,32 @@ xtPublic xtBool xt_pwrite_fmap(XTMapFile memcpy(mm->mm_start + offset, data, size); #endif - xt_rwmutex_unlock(&mm->mm_lock, thd_id); + FILE_MAP_UNLOCK(&mm->mm_lock, thd_id); stat->ts_write += size; return OK; failed: - xt_rwmutex_unlock(&mm->mm_lock, thd_id); + FILE_MAP_UNLOCK(&mm->mm_lock, thd_id); return FAILED; } xtPublic xtBool xt_pread_fmap_4(XTMapFilePtr map, off_t offset, xtWord4 *value, XTIOStatsPtr stat, XTThreadPtr thread) { XTFileMemMapPtr mm = map->mf_memmap; +#ifndef FILE_MAP_USE_PTHREAD_RW xtThreadID thd_id = thread->t_id; +#endif #ifdef DEBUG_TRACE_MAP_IO xt_trace("/* %s */ pbxt_fmap_read_4(\"%s\", %lu, 4);\n", xt_trace_clock_diff(NULL), map->fr_file->fil_path, (u_long) offset); #endif - xt_rwmutex_slock(&mm->mm_lock, thd_id); + if (!map->mf_slock_count) + FILE_MAP_READ_LOCK(&mm->mm_lock, thd_id); if (!mm->mm_start) { - xt_rwmutex_unlock(&mm->mm_lock, thd_id); - xt_rwmutex_xlock(&mm->mm_lock, thd_id); + FILE_MAP_UNLOCK(&mm->mm_lock, thd_id); + FILE_MAP_WRITE_LOCK(&mm->mm_lock, thd_id); if (!fs_remap_file(map, 0, 0, stat)) { - xt_rwmutex_unlock(&mm->mm_lock, thd_id); + FILE_MAP_UNLOCK(&mm->mm_lock, thd_id); return FAILED; } } @@ -1436,7 +1504,7 @@ xtPublic xtBool xt_pread_fmap_4(XTMapFil } __except(EXCEPTION_EXECUTE_HANDLER) { - xt_rwmutex_unlock(&mm->mm_lock, thd_id); + FILE_MAP_UNLOCK(&mm->mm_lock, thd_id); return xt_register_ferrno(XT_REG_CONTEXT, GetExceptionCode(), xt_file_path(map)); } #else @@ -1444,7 +1512,8 @@ xtPublic xtBool xt_pread_fmap_4(XTMapFil #endif } - xt_rwmutex_unlock(&mm->mm_lock, thd_id); + if (!map->mf_slock_count) + FILE_MAP_UNLOCK(&mm->mm_lock, thd_id); stat->ts_read += 4; return OK; } @@ -1452,7 +1521,9 @@ xtPublic xtBool xt_pread_fmap_4(XTMapFil xtPublic xtBool xt_pread_fmap(XTMapFilePtr map, off_t offset, size_t size, size_t min_size, void *data, size_t *red_size, XTIOStatsPtr stat, XTThreadPtr thread) { XTFileMemMapPtr mm = map->mf_memmap; +#ifndef FILE_MAP_USE_PTHREAD_RW xtThreadID thd_id = thread->t_id; +#endif size_t tfer; #ifdef DEBUG_TRACE_MAP_IO @@ -1461,6 +1532,8 @@ xtPublic xtBool xt_pread_fmap(XTMapFileP /* NOTE!! The file map may already be locked, * by a call to xt_lock_fmap_ptr()! * + * 20.05.2009: This problem should be fixed now with mf_slock_count! + * * This can occur during a sequential scan: * xt_pread_fmap() Line 1330 * XTTabCache::tc_read_direct() Line 361 @@ -1491,13 +1564,16 @@ xtPublic xtBool xt_pread_fmap(XTMapFileP * As a result, the slock must be able to handle * nested calls to lock/unlock. */ - xt_rwmutex_slock(&mm->mm_lock, thd_id); + if (!map->mf_slock_count) + FILE_MAP_READ_LOCK(&mm->mm_lock, thd_id); tfer = size; if (!mm->mm_start) { - xt_rwmutex_unlock(&mm->mm_lock, thd_id); - xt_rwmutex_xlock(&mm->mm_lock, thd_id); + FILE_MAP_UNLOCK(&mm->mm_lock, thd_id); + ASSERT_NS(!map->mf_slock_count); + FILE_MAP_WRITE_LOCK(&mm->mm_lock, thd_id); if (!fs_remap_file(map, 0, 0, stat)) { - xt_rwmutex_unlock(&mm->mm_lock, thd_id); + if (!map->mf_slock_count) + FILE_MAP_UNLOCK(&mm->mm_lock, thd_id); return FAILED; } } @@ -1514,7 +1590,8 @@ xtPublic xtBool xt_pread_fmap(XTMapFileP } __except(EXCEPTION_EXECUTE_HANDLER) { - xt_rwmutex_unlock(&mm->mm_lock, thd_id); + if (!map->mf_slock_count) + FILE_MAP_UNLOCK(&mm->mm_lock, thd_id); return xt_register_ferrno(XT_REG_CONTEXT, GetExceptionCode(), xt_file_path(map)); } #else @@ -1522,7 +1599,8 @@ xtPublic xtBool xt_pread_fmap(XTMapFileP #endif } - xt_rwmutex_unlock(&mm->mm_lock, thd_id); + if (!map->mf_slock_count) + FILE_MAP_UNLOCK(&mm->mm_lock, thd_id); if (tfer < min_size) return xt_register_ferrno(XT_REG_CONTEXT, ESPIPE, xt_file_path(map)); @@ -1535,18 +1613,23 @@ xtPublic xtBool xt_pread_fmap(XTMapFileP xtPublic xtBool xt_flush_fmap(XTMapFilePtr map, XTIOStatsPtr stat, XTThreadPtr thread) { XTFileMemMapPtr mm = map->mf_memmap; +#ifndef FILE_MAP_USE_PTHREAD_RW xtThreadID thd_id = thread->t_id; +#endif xtWord8 s; #ifdef DEBUG_TRACE_MAP_IO xt_trace("/* %s */ pbxt_fmap_sync(\"%s\");\n", xt_trace_clock_diff(NULL), map->fr_file->fil_path); #endif - xt_rwmutex_slock(&mm->mm_lock, thd_id); + if (!map->mf_slock_count) + FILE_MAP_READ_LOCK(&mm->mm_lock, thd_id); if (!mm->mm_start) { - xt_rwmutex_unlock(&mm->mm_lock, thd_id); - xt_rwmutex_xlock(&mm->mm_lock, thd_id); + FILE_MAP_UNLOCK(&mm->mm_lock, thd_id); + ASSERT_NS(!map->mf_slock_count); + FILE_MAP_WRITE_LOCK(&mm->mm_lock, thd_id); if (!fs_remap_file(map, 0, 0, stat)) { - xt_rwmutex_unlock(&mm->mm_lock, thd_id); + if (!map->mf_slock_count) + FILE_MAP_UNLOCK(&mm->mm_lock, thd_id); return FAILED; } } @@ -1562,7 +1645,8 @@ xtPublic xtBool xt_flush_fmap(XTMapFileP goto failed; } #endif - xt_rwmutex_unlock(&mm->mm_lock, thd_id); + if (!map->mf_slock_count) + FILE_MAP_UNLOCK(&mm->mm_lock, thd_id); s = stat->ts_flush_start; stat->ts_flush_start = 0; stat->ts_flush_time += xt_trace_clock() - s; @@ -1570,22 +1654,27 @@ xtPublic xtBool xt_flush_fmap(XTMapFileP return OK; failed: - xt_rwmutex_unlock(&mm->mm_lock, thd_id); + if (!map->mf_slock_count) + FILE_MAP_UNLOCK(&mm->mm_lock, thd_id); s = stat->ts_flush_start; stat->ts_flush_start = 0; stat->ts_flush_time += xt_trace_clock() - s; return FAILED; } -xtPublic xtWord1 *xt_lock_fmap_ptr(XTMapFilePtr map, off_t offset, size_t size, XTIOStatsPtr stat, XTThreadPtr XT_UNUSED(thread)) +xtPublic xtWord1 *xt_lock_fmap_ptr(XTMapFilePtr map, off_t offset, size_t size, XTIOStatsPtr stat, XTThreadPtr thread) { XTFileMemMapPtr mm = map->mf_memmap; +#ifndef FILE_MAP_USE_PTHREAD_RW xtThreadID thd_id = thread->t_id; +#endif - xt_rwmutex_slock(&mm->mm_lock, thd_id); + if (!map->mf_slock_count) + FILE_MAP_READ_LOCK(&mm->mm_lock, thd_id); + map->mf_slock_count++; if (!mm->mm_start) { - xt_rwmutex_unlock(&mm->mm_lock, thd_id); - xt_rwmutex_xlock(&mm->mm_lock, thd_id); + FILE_MAP_UNLOCK(&mm->mm_lock, thd_id); + FILE_MAP_WRITE_LOCK(&mm->mm_lock, thd_id); if (!fs_remap_file(map, 0, 0, stat)) goto failed; } @@ -1599,13 +1688,17 @@ xtPublic xtWord1 *xt_lock_fmap_ptr(XTMap return mm->mm_start + offset; failed: - xt_rwmutex_unlock(&mm->mm_lock, thd_id); + map->mf_slock_count--; + if (!map->mf_slock_count) + FILE_MAP_UNLOCK(&mm->mm_lock, thd_id); return NULL; } xtPublic void xt_unlock_fmap_ptr(XTMapFilePtr map, XTThreadPtr thread) { - xt_rwmutex_unlock(&map->mf_memmap->mm_lock, thread->t_id); + map->mf_slock_count--; + if (!map->mf_slock_count) + FILE_MAP_UNLOCK(&map->mf_memmap->mm_lock, thread->t_id); } /* ---------------------------------------------------------------------- === modified file 'storage/pbxt/src/filesys_xt.h' --- a/storage/pbxt/src/filesys_xt.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/filesys_xt.h 2009-08-17 11:12:36 +0000 @@ -76,13 +76,60 @@ xtBool xt_fs_rename(struct XTThread *s #define XT_NULL_FD (-1) #endif +/* Note, this lock must be re-entrant, + * The only lock that satifies this is + * FILE_MAP_USE_RWMUTEX! + * + * 20.05.2009: This problem should be fixed now with mf_slock_count! + * + * The lock need no longer be re-entrant + */ +#ifdef XT_NO_ATOMICS +#define FILE_MAP_USE_PTHREAD_RW +#else +//#define FILE_MAP_USE_RWMUTEX +//#define FILE_MAP_USE_PTHREAD_RW +//#define IDX_USE_SPINXSLOCK +#define FILE_MAP_USE_XSMUTEX +#endif + +#ifdef FILE_MAP_USE_XSMUTEX +#define FILE_MAP_LOCK_TYPE XTXSMutexRec +#define FILE_MAP_INIT_LOCK(s, i) xt_xsmutex_init_with_autoname(s, i) +#define FILE_MAP_FREE_LOCK(s, i) xt_xsmutex_free(s, i) +#define FILE_MAP_READ_LOCK(i, o) xt_xsmutex_slock(i, o) +#define FILE_MAP_WRITE_LOCK(i, o) xt_xsmutex_xlock(i, o) +#define FILE_MAP_UNLOCK(i, o) xt_xsmutex_unlock(i, o) +#elif defined(FILE_MAP_USE_PTHREAD_RW) +#define FILE_MAP_LOCK_TYPE xt_rwlock_type +#define FILE_MAP_INIT_LOCK(s, i) xt_init_rwlock(s, i) +#define FILE_MAP_FREE_LOCK(s, i) xt_free_rwlock(i) +#define FILE_MAP_READ_LOCK(i, o) xt_slock_rwlock_ns(i) +#define FILE_MAP_WRITE_LOCK(i, o) xt_xlock_rwlock_ns(i) +#define FILE_MAP_UNLOCK(i, o) xt_unlock_rwlock_ns(i) +#elif defined(FILE_MAP_USE_RWMUTEX) +#define FILE_MAP_LOCK_TYPE XTRWMutexRec +#define FILE_MAP_INIT_LOCK(s, i) xt_rwmutex_init_with_autoname(s, i) +#define FILE_MAP_FREE_LOCK(s, i) xt_rwmutex_free(s, i) +#define FILE_MAP_READ_LOCK(i, o) xt_rwmutex_slock(i, o) +#define FILE_MAP_WRITE_LOCK(i, o) xt_rwmutex_xlock(i, o) +#define FILE_MAP_UNLOCK(i, o) xt_rwmutex_unlock(i, o) +#elif defined(FILE_MAP_USE_SPINXSLOCK) +#define FILE_MAP_LOCK_TYPE XTSpinXSLockRec +#define FILE_MAP_INIT_LOCK(s, i) xt_spinxslock_init_with_autoname(s, i) +#define FILE_MAP_FREE_LOCK(s, i) xt_spinxslock_free(s, i) +#define FILE_MAP_READ_LOCK(i, o) xt_spinxslock_slock(i, o) +#define FILE_MAP_WRITE_LOCK(i, o) xt_spinxslock_xlock(i, o) +#define FILE_MAP_UNLOCK(i, o) xt_spinxslock_unlock(i, o) +#endif + typedef struct XTFileMemMap { xtWord1 *mm_start; /* The in-memory start of the map. */ #ifdef XT_WIN HANDLE mm_mapdes; #endif off_t mm_length; /* The length of the file map. */ - XTRWMutexRec mm_lock; /* The file map R/W lock. */ + FILE_MAP_LOCK_TYPE mm_lock; /* The file map R/W lock. */ size_t mm_grow_size; /* The amount by which the map file is increased. */ } XTFileMemMapRec, *XTFileMemMapPtr; @@ -127,6 +174,9 @@ xtBool xt_pwrite_file(XTOpenFilePtr of xtBool xt_pread_file(XTOpenFilePtr of, off_t offset, size_t size, size_t min_size, void *data, size_t *red_size, struct XTIOStats *timer, struct XTThread *thread); xtBool xt_flush_file(XTOpenFilePtr of, struct XTIOStats *timer, struct XTThread *thread); +xtBool xt_lock_file_ptr(XTOpenFilePtr of, xtWord1 **data, off_t offset, size_t size, struct XTIOStats *timer, struct XTThread *thread); +void xt_unlock_file_ptr(XTOpenFilePtr of, xtWord1 *data, struct XTThread *thread); + typedef struct XTOpenDir { char *od_path; #ifdef XT_WIN @@ -134,8 +184,14 @@ typedef struct XTOpenDir { WIN32_FIND_DATA od_data; #else char *od_filter; - struct dirent od_entry; DIR *od_dir; + /* WARNING: Solaris requires od_entry.d_name member to have size at least as returned + * by pathconf() function on per-directory basis. This makes it impossible to statically + * pre-set the size. So xt_dir_open on Solaris dynamically allocates space as needed. + * + * This also means that the od_entry member should always be last in the XTOpenDir structure. + */ + struct dirent od_entry; #endif } XTOpenDirRec, *XTOpenDirPtr; @@ -147,6 +203,7 @@ xtBool xt_dir_is_file(struct XTThread off_t xt_dir_file_size(struct XTThread *self, XTOpenDirPtr od); typedef struct XTMapFile : public XTFileRef { + u_int mf_slock_count; XTFileMemMapPtr mf_memmap; } XTMapFileRec, *XTMapFilePtr; === modified file 'storage/pbxt/src/ha_pbxt.cc' --- a/storage/pbxt/src/ha_pbxt.cc 2009-04-02 11:49:57 +0000 +++ b/storage/pbxt/src/ha_pbxt.cc 2009-08-18 07:46:53 +0000 @@ -65,8 +65,8 @@ extern "C" char **session_query(Session #include "heap_xt.h" #include "myxt_xt.h" #include "datadic_xt.h" -#ifdef XT_STREAMING -#include "streaming_xt.h" +#ifdef PBMS_ENABLED +#include "pbms_enabled.h" #endif #include "tabcache_xt.h" #include "systab_xt.h" @@ -91,20 +91,22 @@ extern "C" char **session_query(Session //#define PRINT_STATEMENTS #endif +#ifndef DRIZZLED static handler *pbxt_create_handler(handlerton *hton, TABLE_SHARE *table, MEM_ROOT *mem_root); static int pbxt_init(void *p); static int pbxt_end(void *p); -#ifndef DRIZZLED static int pbxt_panic(handlerton *hton, enum ha_panic_function flag); -#endif static void pbxt_drop_database(handlerton *hton, char *path); static int pbxt_close_connection(handlerton *hton, THD* thd); static int pbxt_commit(handlerton *hton, THD *thd, bool all); static int pbxt_rollback(handlerton *hton, THD *thd, bool all); +#endif static void ha_aquire_exclusive_use(XTThreadPtr self, XTSharePtr share, ha_pbxt *mine); static void ha_release_exclusive_use(XTThreadPtr self, XTSharePtr share); static void ha_close_open_tables(XTThreadPtr self, XTSharePtr share, ha_pbxt *mine); +extern void xt_xres_start_database_recovery(XTThreadPtr self); + #ifdef TRACE_STATEMENTS #ifdef PRINT_STATEMENTS @@ -167,7 +169,7 @@ xtBool pbxt_crash_debug = FALSE; /* Variables for pbxt share methods */ static xt_mutex_type pbxt_database_mutex; // Prevent a database from being opened while it is being dropped static XTHashTabPtr pbxt_share_tables; // Hash used to track open tables -static XTDatabaseHPtr pbxt_database = NULL; // The global open database +XTDatabaseHPtr pbxt_database = NULL; // The global open database static char *pbxt_index_cache_size; static char *pbxt_record_cache_size; static char *pbxt_log_cache_size; @@ -178,6 +180,7 @@ static char *pbxt_checkpoint_frequenc static char *pbxt_data_log_threshold; static char *pbxt_data_file_grow_size; static char *pbxt_row_file_grow_size; +static int pbxt_max_threads; #ifdef DEBUG #define XT_SHARE_LOCK_WAIT 5000 @@ -454,7 +457,7 @@ xtPublic void xt_ha_close_global_databas * operation to make sure it does not occur while * some other thread is doing a "closeall". */ -xtPublic void xt_ha_open_database_of_table(XTThreadPtr self, XTPathStrPtr table_path __attribute__((unused))) +xtPublic void xt_ha_open_database_of_table(XTThreadPtr self, XTPathStrPtr XT_UNUSED(table_path)) { #ifdef XT_USE_GLOBAL_DB if (!self->st_database) { @@ -574,7 +577,7 @@ xtPublic XTThreadPtr xt_ha_thd_to_self(T } /* The first bit is 1. */ -static u_int ha_get_max_bit(MY_BITMAP *map) +static u_int ha_get_max_bit(MX_BITMAP *map) { my_bitmap_map *data_ptr = map->bitmap; my_bitmap_map *end_ptr = map->last_word_ptr; @@ -676,7 +679,7 @@ xtPublic int xt_ha_pbxt_to_mysql_error(i return(-1); // Unknown error } -xtPublic int xt_ha_pbxt_thread_error_for_mysql(THD *thd __attribute__((unused)), const XTThreadPtr self, int ignore_dup_key) +xtPublic int xt_ha_pbxt_thread_error_for_mysql(THD *XT_UNUSED(thd), const XTThreadPtr self, int ignore_dup_key) { int xt_err = self->t_exception.e_xt_err; @@ -965,8 +968,8 @@ static void ha_exit(XTThreadPtr self) /* This may cause the streaming engine to cleanup connections and * tables belonging to this engine. This in turn may require some of * the stuff below (like xt_create_thread() called from pbxt_close_table()! */ -#ifdef XT_STREAMING - xt_exit_streaming(); +#ifdef PBMS_ENABLED + pbms_finalize(); #endif pbxt_call_exit(self); xt_exit_threading(self); @@ -979,9 +982,13 @@ static void ha_exit(XTThreadPtr self) /* * Outout the PBXT status. Return FALSE on error. */ -static bool pbxt_show_status(handlerton *hton __attribute__((unused)), THD* thd, +#ifdef DRIZZLED +bool PBXTStorageEngine::show_status(Session *thd, stat_print_fn *stat_print, enum ha_stat_type) +#else +static bool pbxt_show_status(handlerton *XT_UNUSED(hton), THD* thd, stat_print_fn* stat_print, - enum ha_stat_type stat_type __attribute__((unused))) + enum ha_stat_type XT_UNUSED(stat_type)) +#endif { XTThreadPtr self; int err = 0; @@ -997,6 +1004,9 @@ static bool pbxt_show_status(handlerton xt_trace("// %s - dump\n", xt_trace_clock_diff(NULL)); xt_dump_trace(); #endif +#ifdef XT_TRACK_CONNECTIONS + xt_dump_conn_tracking(); +#endif try_(a) { myxt_get_status(self, &strbuf); @@ -1020,14 +1030,18 @@ static bool pbxt_show_status(handlerton * * return 1 on error, else 0. */ +#ifdef DRIZZLED +static int pbxt_init(PluginRegistry &registry) +#else static int pbxt_init(void *p) +#endif { int init_err = 0; XT_TRACE_CALL(); if (sizeof(xtWordPS) != sizeof(void *)) { - printf("PBXT: This won't work, I require that sizeof(xtWordPS) != sizeof(void *)!\n"); + printf("PBXT: This won't work, I require that sizeof(xtWordPS) == sizeof(void *)!\n"); XT_RETURN(1); } @@ -1045,28 +1059,31 @@ static int pbxt_init(void *p) xt_p_mutex_init_with_autoname(&pbxt_database_mutex, NULL); +#ifdef DRIZZLED + pbxt_hton= new PBXTStorageEngine(std::string("PBXT")); + registry.add(pbxt_hton); +#else pbxt_hton = (handlerton *) p; pbxt_hton->state = SHOW_OPTION_YES; -#ifndef DRIZZLED pbxt_hton->db_type = DB_TYPE_PBXT; // Wow! I have my own! -#endif pbxt_hton->close_connection = pbxt_close_connection; /* close_connection, cleanup thread related data. */ pbxt_hton->commit = pbxt_commit; /* commit */ pbxt_hton->rollback = pbxt_rollback; /* rollback */ pbxt_hton->create = pbxt_create_handler; /* Create a new handler */ pbxt_hton->drop_database = pbxt_drop_database; /* Drop a database */ -#ifndef DRIZZLED pbxt_hton->panic = pbxt_panic; /* Panic call */ -#endif pbxt_hton->show_status = pbxt_show_status; pbxt_hton->flags = HTON_NO_FLAGS; /* HTON_CAN_RECREATE - Without this flags TRUNCATE uses delete_all_rows() */ - +#endif if (!xt_init_logging()) /* Initialize logging */ goto error_1; -#ifdef XT_STREAMING - if (!xt_init_streaming()) +#ifdef PBMS_ENABLED + PBMSResultRec result; + if (!pbms_initialize("PBXT", false, &result)) { + xt_logf(XT_NT_ERROR, "pbms_initialize() Error: %s", result.mr_message); goto error_2; + } #endif if (!xt_init_memory()) /* Initialize memory */ @@ -1082,9 +1099,13 @@ static int pbxt_init(void *p) * +1 Free'er thread * +1 Temporary thread (e.g. TempForClose, TempForEnd) */ - self = xt_init_threading(max_connections + 7); /* Create the main self: */ +#ifndef DRIZZLED + if (pbxt_max_threads == 0) + pbxt_max_threads = max_connections + 7; +#endif + self = xt_init_threading(pbxt_max_threads); /* Create the main self: */ if (!self) - goto error_4; + goto error_3; pbxt_inited = true; @@ -1111,7 +1132,7 @@ static int pbxt_init(void *p) ASSERT(!pbxt_database); { THD *curr_thd = current_thd; - THD *thd = curr_thd; + THD *thd = NULL; #ifndef DRIZZLED extern myxt_mutex_t LOCK_plugin; @@ -1148,21 +1169,23 @@ static int pbxt_init(void *p) xt_throw(self); } - xt_open_database(self, mysql_real_data_home, TRUE); - pbxt_database = self->st_database; - xt_heap_reference(self, pbxt_database); + xt_xres_start_database_recovery(self); } catch_(b) { - if (!curr_thd && thd) - myxt_destroy_thread(thd, FALSE); -#ifndef DRIZZLED - myxt_mutex_lock(&LOCK_plugin); -#endif - xt_throw(self); + /* It is possible that the error was reset by cleanup code. + * Set a generic error code in that case. + */ + /* PMC - This is not necessary in because exceptions are + * now preserved, in exception handler cleanup. + */ + if (!self->t_exception.e_xt_err) + xt_register_error(XT_REG_CONTEXT, XT_SYSTEM_ERROR, 0, "Initialization failed"); + xt_log_exception(self, &self->t_exception, XT_LOG_DEFAULT); + init_err = 1; } cont_(b); - if (!curr_thd) + if (thd) myxt_destroy_thread(thd, FALSE); #ifndef DRIZZLED myxt_mutex_lock(&LOCK_plugin); @@ -1205,32 +1228,37 @@ static int pbxt_init(void *p) * I have to stop the freeer here because it was * started before opening the database. */ - pbxt_call_exit(self); - pbxt_inited = FALSE; - xt_exit_threading(self); - goto error_4; + + /* {FREEER-HANG-ON-INIT-ERROR} + * pbxt_init is called with LOCK_plugin and if it fails and tries to exit + * the freeer here it hangs because the freeer calls THD::~THD which tries + * to aquire the same lock and hangs. OTOH MySQL calls pbxt_end() after + * an unsuccessful call to pbxt_init, so we defer cleaup, except + * releasing 'self' + */ + xt_free_thread(self); + goto error_3; } xt_free_thread(self); } XT_RETURN(init_err); - error_4: - xt_exit_memory(); - error_3: -#ifdef XT_STREAMING - xt_exit_streaming(); +#ifdef PBMS_ENABLED + pbms_finalize(); error_2: #endif - xt_exit_logging(); error_1: - xt_p_mutex_destroy(&pbxt_database_mutex); XT_RETURN(1); } -static int pbxt_end(void *p __attribute__((unused))) +#ifdef DRIZZLED +static int pbxt_end(PluginRegistry &registry) +#else +static int pbxt_end(void *) +#endif { XTThreadPtr self; int err = 0; @@ -1241,7 +1269,7 @@ static int pbxt_end(void *p __attribute_ XTExceptionRec e; /* This flag also means "shutting down". */ - pbxt_inited = FALSE; + pbxt_inited = FALSE; self = xt_create_thread("TempForEnd", FALSE, TRUE, &e); if (self) { self->t_main = TRUE; @@ -1249,6 +1277,9 @@ static int pbxt_end(void *p __attribute_ } } +#ifdef DRIZZLED + registry.remove(pbxt_hton); +#endif XT_RETURN(err); } @@ -1262,12 +1293,15 @@ static int pbxt_panic(handlerton *hton, /* * Kill the PBXT thread associated with the MySQL thread. */ +#ifdef DRIZZLED +int PBXTStorageEngine::close_connection(Session *thd) +{ + PBXTStorageEngine * const hton = this; +#else static int pbxt_close_connection(handlerton *hton, THD* thd) { - XTThreadPtr self; -#ifdef XT_STREAMING - XTExceptionRec e; #endif + XTThreadPtr self; XT_TRACE_CALL(); if ((self = (XTThreadPtr) *thd_ha_data(thd, hton))) { @@ -1278,10 +1312,6 @@ static int pbxt_close_connection(handler xt_set_self(self); xt_free_thread(self); } -#ifdef XT_STREAMING - if (!xt_pbms_close_connection((void *) thd, &e)) - xt_log_exception(NULL, &e, XT_LOG_DEFAULT); -#endif return 0; } @@ -1290,7 +1320,11 @@ static int pbxt_close_connection(handler * when the last PBXT table was removed from the * database. */ -static void pbxt_drop_database(handlerton *hton __attribute__((unused)), char *path __attribute__((unused))) +#ifdef DRIZZLED +void PBXTStorageEngine::drop_database(char *) +#else +static void pbxt_drop_database(handlerton *XT_UNUSED(hton), char *XT_UNUSED(path)) +#endif { XT_TRACE_CALL(); } @@ -1317,8 +1351,14 @@ static void pbxt_drop_database(handlerto * pbxt_thr is a pointer the the PBXT thread structure. * */ +#ifdef DRIZZLED +int PBXTStorageEngine::commit(Session *thd, bool all) +{ + PBXTStorageEngine * const hton = this; +#else static int pbxt_commit(handlerton *hton, THD *thd, bool all) { +#endif int err = 0; XTThreadPtr self; @@ -1343,8 +1383,14 @@ static int pbxt_commit(handlerton *hton, return err; } +#ifdef DRIZZLED +int PBXTStorageEngine::rollback(Session *thd, bool all) +{ + PBXTStorageEngine * const hton = this; +#else static int pbxt_rollback(handlerton *hton, THD *thd, bool all) { +#endif int err = 0; XTThreadPtr self; @@ -1377,8 +1423,14 @@ static int pbxt_rollback(handlerton *hto return 0; } +#ifdef DRIZZLED +handler *PBXTStorageEngine::create(TABLE_SHARE *table, MEM_ROOT *mem_root) +{ + PBXTStorageEngine * const hton = this; +#else static handler *pbxt_create_handler(handlerton *hton, TABLE_SHARE *table, MEM_ROOT *mem_root) { +#endif if (table && XTSystemTableShare::isSystemTable(table->path.str)) return new (mem_root) ha_xtsys(hton, table); else @@ -1513,7 +1565,11 @@ static void ha_close_open_tables(XTThrea freer_(); // xt_unlock_mutex(share->sh_ex_mutex) } -static void ha_release_exclusive_use(XTThreadPtr self __attribute__((unused)), XTSharePtr share) +#ifdef PBXT_ALLOW_PRINTING +static void ha_release_exclusive_use(XTThreadPtr self, XTSharePtr share) +#else +static void ha_release_exclusive_use(XTThreadPtr XT_UNUSED(self), XTSharePtr share) +#endif { XT_PRINT1(self, "ha_release_exclusive_use %s PBXT X UNLOCK\n", share->sh_table_path->ps_path); xt_lock_mutex_ns((xt_mutex_type *) share->sh_ex_mutex); @@ -1629,11 +1685,23 @@ ST_FIELD_INFO pbxt_statistics_fields_inf { 0, 0, MYSQL_TYPE_STRING, 0, 0, 0, SKIP_OPEN_TABLE} }; +#ifdef DRIZZLED +static InfoSchemaTable *pbxt_statistics_table; + +int pbxt_init_statitics(PluginRegistry &registry) +#else int pbxt_init_statitics(void *p) +#endif { - ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE *) p; - schema->fields_info = pbxt_statistics_fields_info; - schema->fill_table = pbxt_statistics_fill_table; +#ifdef DRIZZLED + pbxt_statistics_table = (InfoSchemaTable *)xt_calloc_ns(sizeof(InfoSchemaTable)); + pbxt_statistics_table->table_name= "PBXT_STATISTICS"; + registry.add(pbxt_statistics_table); +#else + ST_SCHEMA_TABLE *pbxt_statistics_table = (ST_SCHEMA_TABLE *) p; +#endif + pbxt_statistics_table->fields_info = pbxt_statistics_fields_info; + pbxt_statistics_table->fill_table = pbxt_statistics_fill_table; #if defined(XT_WIN) && defined(XT_COREDUMP) void register_crash_filter(); @@ -1645,8 +1713,16 @@ int pbxt_init_statitics(void *p) return 0; } -int pbxt_exit_statitics(void *p __attribute__((unused))) +#ifdef DRIZZLED +int pbxt_exit_statitics(PluginRegistry &registry) +#else +int pbxt_exit_statitics(void *XT_UNUSED(p)) +#endif { +#ifdef DRIZZLED + registry.remove(pbxt_statistics_table); + xt_free_ns(pbxt_statistics_table); +#endif return(0); } @@ -1765,7 +1841,7 @@ MX_TABLE_TYPES_T ha_pbxt::table_flags() */ #define FLAGS_ARE_READ_DYNAMICALLY -MX_ULONG_T ha_pbxt::index_flags(uint inx __attribute__((unused)), uint part __attribute__((unused)), bool all_parts __attribute__((unused))) const +MX_ULONG_T ha_pbxt::index_flags(uint XT_UNUSED(inx), uint XT_UNUSED(part), bool XT_UNUSED(all_parts)) const { /* It would be nice if the dynamic version of this function works, * but it does not. MySQL loads this information when the table is openned, @@ -1876,7 +1952,7 @@ void ha_pbxt::internal_close(THD *thd, s * Called from handler.cc by handler::ha_open(). The server opens all tables by * calling ha_open() which then calls the handler specific open(). */ -int ha_pbxt::open(const char *table_path, int mode __attribute__((unused)), uint test_if_locked __attribute__((unused))) +int ha_pbxt::open(const char *table_path, int XT_UNUSED(mode), uint XT_UNUSED(test_if_locked)) { THD *thd = current_thd; int err = 0; @@ -2104,9 +2180,9 @@ void ha_pbxt::init_auto_increment(xtWord } void ha_pbxt::get_auto_increment(MX_ULONGLONG_T offset, MX_ULONGLONG_T increment, - MX_ULONGLONG_T nb_desired_values __attribute__((unused)), + MX_ULONGLONG_T XT_UNUSED(nb_desired_values), MX_ULONGLONG_T *first_value, - MX_ULONGLONG_T *nb_reserved_values __attribute__((unused))) + MX_ULONGLONG_T *nb_reserved_values) { register XTTableHPtr tab; MX_ULONGLONG_T nr, nr_less_inc; @@ -2225,6 +2301,14 @@ int ha_pbxt::write_row(byte *buf) XT_PRINT1(pb_open_tab->ot_thread, "ha_pbxt::write_row %s\n", pb_share->sh_table_path->ps_path); XT_DISABLED_TRACE(("INSERT tx=%d val=%d\n", (int) pb_open_tab->ot_thread->st_xact_data->xd_start_xn_id, (int) XT_GET_DISK_4(&buf[1]))); //statistic_increment(ha_write_count,&LOCK_status); +#ifdef PBMS_ENABLED + PBMSResultRec result; + err = pbms_write_row_blobs(table, buf, &result); + if (err) { + xt_logf(XT_NT_ERROR, "pbms_write_row_blobs() Error: %s", result.mr_message); + return err; + } +#endif /* GOTCHA: I have a huge problem with the transaction statement. * It is not ALWAYS committed (I mean ha_commit_trans() is @@ -2256,7 +2340,8 @@ int ha_pbxt::write_row(byte *buf) int update_err = update_auto_increment(); if (update_err) { ha_log_pbxt_thread_error_for_mysql(pb_ignore_dup_key); - return update_err; + err = update_err; + goto done; } set_auto_increment(table->next_number_field); } @@ -2274,6 +2359,10 @@ int ha_pbxt::write_row(byte *buf) pb_open_tab->ot_thread->st_update_id++; } + done: +#ifdef PBMS_ENABLED + pbms_completed(table, (err == 0)); +#endif return err; } @@ -2347,6 +2436,21 @@ int ha_pbxt::update_row(const byte * old if (table->timestamp_field_type & TIMESTAMP_AUTO_SET_ON_UPDATE) table->timestamp_field->set_time(); +#ifdef PBMS_ENABLED + PBMSResultRec result; + + err = pbms_delete_row_blobs(table, old_data, &result); + if (err) { + xt_logf(XT_NT_ERROR, "update_row:pbms_delete_row_blobs() Error: %s", result.mr_message); + return err; + } + err = pbms_write_row_blobs(table, new_data, &result); + if (err) { + xt_logf(XT_NT_ERROR, "update_row:pbms_write_row_blobs() Error: %s", result.mr_message); + goto pbms_done; + } +#endif + /* GOTCHA: We need to check the auto-increment value on update * because of the following test (which fails for InnoDB) - * auto_increment.test: @@ -2369,6 +2473,11 @@ int ha_pbxt::update_row(const byte * old err = ha_log_pbxt_thread_error_for_mysql(pb_ignore_dup_key); pb_open_tab->ot_table->tab_locks.xt_remove_temp_lock(pb_open_tab, TRUE); + +#ifdef PBMS_ENABLED + pbms_done: + pbms_completed(table, (err == 0)); +#endif return err; } @@ -2392,6 +2501,16 @@ int ha_pbxt::delete_row(const byte * buf XT_DISABLED_TRACE(("DELETE tx=%d val=%d\n", (int) pb_open_tab->ot_thread->st_xact_data->xd_start_xn_id, (int) XT_GET_DISK_4(&buf[1]))); //statistic_increment(ha_delete_count,&LOCK_status); +#ifdef PBMS_ENABLED + PBMSResultRec result; + + err = pbms_delete_row_blobs(table, buf, &result); + if (err) { + xt_logf(XT_NT_ERROR, "pbms_delete_row_blobs() Error: %s", result.mr_message); + return err; + } +#endif + if (!pb_open_tab->ot_thread->st_stat_trans) { trans_register_ha(pb_mysql_thd, FALSE, pbxt_hton); XT_PRINT0(pb_open_tab->ot_thread, "ha_pbxt::delete_row trans_register_ha all=FALSE\n"); @@ -2405,6 +2524,9 @@ int ha_pbxt::delete_row(const byte * buf pb_open_tab->ot_table->tab_locks.xt_remove_temp_lock(pb_open_tab, TRUE); +#ifdef PBMS_ENABLED + pbms_completed(table, (err == 0)); +#endif return err; } @@ -2491,7 +2613,7 @@ int ha_pbxt::delete_row(const byte * buf * commit; */ -int ha_pbxt::xt_index_in_range(register XTOpenTablePtr ot __attribute__((unused)), register XTIndexPtr ind, +int ha_pbxt::xt_index_in_range(register XTOpenTablePtr XT_UNUSED(ot), register XTIndexPtr ind, register XTIdxSearchKeyPtr search_key, xtWord1 *buf) { /* If search key is given, this means we want an exact match. */ @@ -2698,7 +2820,7 @@ int ha_pbxt::xt_index_prev_read(XTOpenTa return ha_log_pbxt_thread_error_for_mysql(FALSE); } -int ha_pbxt::index_init(uint idx, bool sorted __attribute__((unused))) +int ha_pbxt::index_init(uint idx, bool XT_UNUSED(sorted)) { XTIndexPtr ind; @@ -2715,7 +2837,8 @@ int ha_pbxt::index_init(uint idx, bool s /* The number of columns required: */ if (pb_open_tab->ot_is_modify) { - pb_open_tab->ot_cols_req = table->read_set->n_bits; + + pb_open_tab->ot_cols_req = table->read_set->MX_BIT_SIZE(); #ifdef XT_PRINT_INDEX_OPT ind = (XTIndexPtr) pb_share->sh_dic_keys[idx]; @@ -2764,10 +2887,10 @@ int ha_pbxt::index_init(uint idx, bool s * seem to have this problem! */ ind = (XTIndexPtr) pb_share->sh_dic_keys[idx]; - if (bitmap_is_subset(table->read_set, &ind->mi_col_map)) + if (MX_BIT_IS_SUBSET(table->read_set, &ind->mi_col_map)) pb_key_read = TRUE; #ifdef XT_PRINT_INDEX_OPT - printf("index_init %s index %d cols req=%d/%d read_bits=%X write_bits=%X index_bits=%X converage=%d\n", pb_open_tab->ot_table->tab_name->ps_path, (int) idx, pb_open_tab->ot_cols_req, table->read_set->n_bits, (int) *table->read_set->bitmap, (int) *table->write_set->bitmap, (int) *ind->mi_col_map.bitmap, (int) (bitmap_is_subset(table->read_set, &ind->mi_col_map) != 0)); + printf("index_init %s index %d cols req=%d/%d read_bits=%X write_bits=%X index_bits=%X converage=%d\n", pb_open_tab->ot_table->tab_name->ps_path, (int) idx, pb_open_tab->ot_cols_req, table->read_set->MX_BIT_SIZE(), (int) *table->read_set->bitmap, (int) *table->write_set->bitmap, (int) *ind->mi_col_map.bitmap, (int) (MX_BIT_IS_SUBSET(table->read_set, &ind->mi_col_map) != 0)); #endif } @@ -2845,7 +2968,7 @@ void ha_return_row(XTOpenTablePtr ot, u_ } #endif -int ha_pbxt::index_read_xt(byte * buf, uint idx, const byte *key, uint key_len __attribute__((unused)), enum ha_rkey_function find_flag __attribute__((unused))) +int ha_pbxt::index_read_xt(byte * buf, uint idx, const byte *key, uint key_len, enum ha_rkey_function find_flag) { int err = 0; XTIndexPtr ind; @@ -2887,9 +3010,12 @@ int ha_pbxt::index_read_xt(byte * buf, u xt_idx_prep_key(ind, &search_key, ((find_flag == HA_READ_AFTER_KEY) ? XT_SEARCH_AFTER_KEY : 0) | prefix, (xtWord1 *) key, key_len); if (!xt_idx_search(pb_open_tab, ind, &search_key)) err = ha_log_pbxt_thread_error_for_mysql(pb_ignore_dup_key); - else + else { err = xt_index_next_read(pb_open_tab, ind, pb_key_read, (find_flag == HA_READ_KEY_EXACT || find_flag == HA_READ_PREFIX) ? &search_key : NULL, buf); + if (err == HA_ERR_END_OF_FILE && find_flag == HA_READ_AFTER_KEY) + err = HA_ERR_KEY_NOT_FOUND; + } break; } @@ -2913,13 +3039,13 @@ int ha_pbxt::index_read_xt(byte * buf, u * row if available. If the key value is null, begin at the first key of the * index. */ -int ha_pbxt::index_read(byte * buf, const byte * key, uint key_len __attribute__((unused)), enum ha_rkey_function find_flag __attribute__((unused))) +int ha_pbxt::index_read(byte * buf, const byte * key, uint key_len, enum ha_rkey_function find_flag) { //statistic_increment(ha_read_key_count,&LOCK_status); return index_read_xt(buf, active_index, key, key_len, find_flag); } -int ha_pbxt::index_read_idx(byte * buf, uint idx, const byte *key, uint key_len __attribute__((unused)), enum ha_rkey_function find_flag __attribute__((unused))) +int ha_pbxt::index_read_idx(byte * buf, uint idx, const byte *key, uint key_len, enum ha_rkey_function find_flag) { //statistic_increment(ha_read_key_count,&LOCK_status); return index_read_xt(buf, idx, key, key_len, find_flag); @@ -3147,9 +3273,24 @@ int ha_pbxt::rnd_init(bool scan) XT_PRINT1(pb_open_tab->ot_thread, "ha_pbxt::rnd_init %s\n", pb_share->sh_table_path->ps_path); XT_DISABLED_TRACE(("seq scan tx=%d\n", (int) pb_open_tab->ot_thread->st_xact_data->xd_start_xn_id)); + /* Call xt_tab_seq_exit() to make sure the resources used by the previous + * scan are freed. In particular make sure cache page ref count is decremented. + * This is needed as rnd_init() can be called mulitple times w/o matching calls + * to rnd_end(). Our experience is that currently this is done in queries like: + * + * SELECT t1.c1,t2.c1 FROM t1 LEFT JOIN t2 USING (c1); + * UPDATE t1 LEFT JOIN t2 USING (c1) SET t1.c1 = t2.c1 WHERE t1.c1 = t2.c1; + * + * when scanning inner tables. It is important to understand that in such case + * multiple calls to rnd_init() are not semantically equal to a new query. For + * example we cannot make row locks permanent as we do in rnd_end(), as + * ha_pbxt::unlock_row still can be called. + */ + xt_tab_seq_exit(pb_open_tab); + /* The number of columns required: */ if (pb_open_tab->ot_is_modify) - pb_open_tab->ot_cols_req = table->read_set->n_bits; + pb_open_tab->ot_cols_req = table->read_set->MX_BIT_SIZE(); else { pb_open_tab->ot_cols_req = ha_get_max_bit(table->read_set); @@ -3243,7 +3384,7 @@ int ha_pbxt::rnd_next(byte *buf) * * Called from filesort.cc, sql_select.cc, sql_delete.cc and sql_update.cc. */ -void ha_pbxt::position(const byte *record __attribute__((unused))) +void ha_pbxt::position(const byte *XT_UNUSED(record)) { XT_TRACE_CALL(); ASSERT_NS(pb_ex_in_use); @@ -3383,7 +3524,7 @@ int ha_pbxt::info(uint flag) if (flag & HA_STATUS_VARIABLE) { stats.deleted = ot->ot_table->tab_row_fnum; stats.records = (ha_rows) (ot->ot_table->tab_row_eof_id - 1 - stats.deleted); - stats.data_file_length = ot->ot_table->tab_rec_eof_id; + stats.data_file_length = xt_rec_id_to_rec_offset(ot->ot_table, ot->ot_table->tab_rec_eof_id); stats.index_file_length = xt_ind_node_to_offset(ot->ot_table, ot->ot_table->tab_ind_eof); stats.delete_length = ot->ot_table->tab_rec_fnum * ot->ot_rec_size; //check_time = info.check_time; @@ -3434,10 +3575,15 @@ int ha_pbxt::info(uint flag) #endif #endif // SAFE_MUTEX +#ifdef DRIZZLED + set_prefix(share->keys_in_use, share->keys); + share->keys_for_keyread&= share->keys_in_use; +#else share->keys_in_use.set_prefix(share->keys); //share->keys_in_use.intersect_extended(info.key_map); share->keys_for_keyread.intersect(share->keys_in_use); //share->db_record_offset = info.record_offset; +#endif for (u_int i = 0; i < share->keys; i++) { ind = pb_share->sh_dic_keys[i]; @@ -3445,7 +3591,7 @@ int ha_pbxt::info(uint flag) if (ind->mi_seg_count == 1 && (ind->mi_flags & HA_NOSAME)) rec_per_key = 1; else { - + rec_per_key = 1; } for (u_int j = 0; j < table->key_info[i].key_parts; j++) table->key_info[i].rec_per_key[j] = (ulong) rec_per_key; @@ -3570,6 +3716,8 @@ int ha_pbxt::extra(enum ha_extra_functio if (pb_open_tab) pb_open_tab->ot_table->tab_locks.xt_make_lock_permanent(pb_open_tab, &self->st_lock_list); } + if (pb_open_tab) + pb_open_tab->ot_for_update = 0; break; case HA_EXTRA_KEYREAD: /* This means we so not need to read the entire record. */ @@ -3706,6 +3854,12 @@ int ha_pbxt::delete_all_rows() */ ha_close_share(self, pb_share); + /* MySQL documentation requires us to reset auto increment value to 1 + * on truncate even if the table was created with a different value. + * This is also consistent with other engines. + */ + dic.dic_min_auto_inc = 1; + xt_create_table(self, (XTPathStrPtr) path, &dic); if (!pb_table_locked) freer_(); // ha_release_exclusive_use(pb_share) @@ -3737,7 +3891,7 @@ int ha_pbxt::delete_all_rows() * now agree with the MyISAM strategy. * */ -int ha_pbxt::analyze(THD *thd __attribute__((unused)), HA_CHECK_OPT *check_opt __attribute__((unused))) +int ha_pbxt::analyze(THD *thd, HA_CHECK_OPT *XT_UNUSED(check_opt)) { int err = 0; XTDatabaseHPtr db; @@ -3819,7 +3973,7 @@ int ha_pbxt::analyze(THD *thd __attribut XT_RETURN(err); } -int ha_pbxt::repair(THD *thd __attribute__((unused)), HA_CHECK_OPT *check_opt __attribute__((unused))) +int ha_pbxt::repair(THD *XT_UNUSED(thd), HA_CHECK_OPT *XT_UNUSED(check_opt)) { return(HA_ADMIN_TRY_ALTER); } @@ -3828,7 +3982,7 @@ int ha_pbxt::repair(THD *thd __attribute * This is mapped to "ALTER TABLE tablename TYPE=PBXT", which rebuilds * the table in MySQL. */ -int ha_pbxt::optimize(THD *thd __attribute__((unused)), HA_CHECK_OPT *check_opt __attribute__((unused))) +int ha_pbxt::optimize(THD *XT_UNUSED(thd), HA_CHECK_OPT *XT_UNUSED(check_opt)) { return(HA_ADMIN_TRY_ALTER); } @@ -3837,7 +3991,7 @@ int ha_pbxt::optimize(THD *thd __attribu extern int pbxt_mysql_trace_on; #endif -int ha_pbxt::check(THD* thd, HA_CHECK_OPT* check_opt __attribute__((unused))) +int ha_pbxt::check(THD* thd, HA_CHECK_OPT* XT_UNUSED(check_opt)) { int err = 0; XTThreadPtr self; @@ -3993,8 +4147,10 @@ xtPublic int ha_pbxt::external_lock(THD * (or update statement) just saw. */ if (pb_open_tab) { - if (pb_open_tab->ot_for_update) + if (pb_open_tab->ot_for_update) { self->st_visible_time = self->st_database->db_xn_end_time; + pb_open_tab->ot_for_update = 0; + } if (pb_share->sh_recalc_selectivity) { if ((pb_share->sh_table->tab_row_eof_id - 1 - pb_share->sh_table->tab_row_fnum) >= 200) { @@ -4079,10 +4235,15 @@ xtPublic int ha_pbxt::external_lock(THD pb_open_tab->ot_is_modify = FALSE; if ((pb_open_tab->ot_for_update = (lock_type == F_WRLCK))) { switch ((int) thd_sql_command(thd)) { - case SQLCOM_UPDATE: - case SQLCOM_UPDATE_MULTI: case SQLCOM_DELETE: case SQLCOM_DELETE_MULTI: + /* turn DELETE IGNORE into normal DELETE. The IGNORE option causes problems because + * when a record is deleted we add an xlog record which we cannot "rollback" later + * when we find that an FK-constraint has failed. + */ + thd->lex->ignore = false; + case SQLCOM_UPDATE: + case SQLCOM_UPDATE_MULTI: case SQLCOM_REPLACE: case SQLCOM_REPLACE_SELECT: case SQLCOM_INSERT: @@ -4290,7 +4451,9 @@ int ha_pbxt::start_stmt(THD *thd, thr_lo pb_open_tab->ot_for_update = (lock_type != TL_READ && lock_type != TL_READ_WITH_SHARED_LOCKS && +#ifndef DRIZZLED lock_type != TL_READ_HIGH_PRIORITY && +#endif lock_type != TL_READ_NO_INSERT); pb_open_tab->ot_is_modify = FALSE; if (pb_open_tab->ot_for_update) { @@ -4557,9 +4720,12 @@ int ha_pbxt::delete_table(const char *ta { THD *thd = current_thd; int err = 0; - XTThreadPtr self; + XTThreadPtr self = NULL; XTSharePtr share; + STAT_TRACE(self, *thd_query(thd)); + XT_PRINT1(self, "ha_pbxt::delete_table %s\n", table_path); + if (XTSystemTableShare::isSystemTable(table_path)) return delete_system_table(table_path); @@ -4568,9 +4734,6 @@ int ha_pbxt::delete_table(const char *ta self->st_ignore_fkeys = (thd_test_options(thd, OPTION_NO_FOREIGN_KEY_CHECKS)) != 0; - STAT_TRACE(self, *thd_query(thd)); - XT_PRINT1(self, "ha_pbxt::delete_table %s\n", table_path); - try_(a) { xt_ha_open_database_of_table(self, (XTPathStrPtr) table_path); @@ -4586,16 +4749,23 @@ int ha_pbxt::delete_table(const char *ta pushr_(ha_release_exclusive_use, share); ha_close_open_tables(self, share, NULL); - xt_drop_table(self, (XTPathStrPtr) table_path); + xt_drop_table(self, (XTPathStrPtr) table_path, thd_sql_command(thd) == SQLCOM_DROP_DB); freer_(); // ha_release_exclusive_use(share) freer_(); // ha_unget_share(share) } catch_(b) { - /* If the table does not exist, just log the error and continue... */ + /* In MySQL if the table does not exist, just log the error and continue. This is + * needed to delete table in the case when CREATE TABLE fails and no PBXT disk + * structures were created. + * Drizzle unlike MySQL iterates over all handlers and tries to delete table. It + * stops after when a handler returns TRUE, so in Drizzle we need to report error. + */ +#ifndef DRIZZLED if (self->t_exception.e_xt_err == XT_ERR_TABLE_NOT_FOUND) xt_log_and_clear_exception(self); else +#endif throw_(); } cont_(b); @@ -4619,8 +4789,25 @@ int ha_pbxt::delete_table(const char *ta } catch_(a) { err = xt_ha_pbxt_thread_error_for_mysql(thd, self, pb_ignore_dup_key); +#ifdef DRIZZLED + if (err == HA_ERR_NO_SUCH_TABLE) + err = ENOENT; +#endif } cont_(a); + +#ifdef PBMS_ENABLED + /* Call pbms_delete_table_with_blobs() last because it cannot be undone. */ + if (!err) { + PBMSResultRec result; + + if (pbms_delete_table_with_blobs(table_path, &result)) { + xt_logf(XT_NT_WARNING, "pbms_delete_table_with_blobs() Error: %s", result.mr_message); + } + + pbms_completed(NULL, true); + } +#endif return err; } @@ -4681,6 +4868,16 @@ int ha_pbxt::rename_table(const char *fr XT_PRINT2(self, "ha_pbxt::rename_table %s -> %s\n", from, to); +#ifdef PBMS_ENABLED + PBMSResultRec result; + + err = pbms_rename_table_with_blobs(from, to, &result); + if (err) { + xt_logf(XT_NT_ERROR, "pbms_rename_table_with_blobs() Error: %s", result.mr_message); + return err; + } +#endif + try_(a) { xt_ha_open_database_of_table(self, (XTPathStrPtr) to); to_db = self->st_database; @@ -4709,10 +4906,6 @@ int ha_pbxt::rename_table(const char *fr freer_(); // ha_release_exclusive_use(share) freer_(); // ha_unget_share(share) -#ifdef XT_STREAMING - /* PBMS remove the table? */ - xt_pbms_rename_table(from, to); -#endif /* * If there are no more PBXT tables in the database, we * "drop the database", which deletes all PBXT resources @@ -4732,11 +4925,15 @@ int ha_pbxt::rename_table(const char *fr err = xt_ha_pbxt_thread_error_for_mysql(thd, self, pb_ignore_dup_key); } cont_(a); + +#ifdef PBMS_ENABLED + pbms_completed(NULL, (err == 0)); +#endif XT_RETURN(err); } -int ha_pbxt::rename_system_table(const char *from __attribute__((unused)), const char *to __attribute__((unused))) +int ha_pbxt::rename_system_table(const char *XT_UNUSED(from), const char *XT_UNUSED(to)) { return ER_NOT_SUPPORTED_YET; } @@ -4771,7 +4968,7 @@ double ha_pbxt::scan_time() /* * The next method will never be called if you do not implement indexes. */ -double ha_pbxt::read_time(uint index __attribute__((unused)), uint ranges, ha_rows rows) +double ha_pbxt::read_time(uint XT_UNUSED(index), uint ranges, ha_rows rows) { double result = rows2double(ranges+rows); return result; @@ -4945,7 +5142,7 @@ void ha_pbxt::free_foreign_key_create_in xt_free(NULL, str); } -bool ha_pbxt::get_error_message(int error __attribute__((unused)), String *buf) +bool ha_pbxt::get_error_message(int XT_UNUSED(error), String *buf) { THD *thd = current_thd; int err = 0; @@ -5104,9 +5301,9 @@ struct st_mysql_sys_var #endif #ifdef USE_CONST_SAVE -static void pbxt_record_cache_size_func(THD *thd __attribute__((unused)), struct st_mysql_sys_var *var, void *tgt, const void *save) +static void pbxt_record_cache_size_func(THD *XT_UNUSED(thd), struct st_mysql_sys_var *var, void *tgt, const void *save) #else -static void pbxt_record_cache_size_func(THD *thd __attribute__((unused)), struct st_mysql_sys_var *var, void *tgt, void *save) +static void pbxt_record_cache_size_func(THD *XT_UNUSED(thd), struct st_mysql_sys_var *var, void *tgt, void *save) #endif { xtInt8 record_cache_size; @@ -5215,6 +5412,18 @@ static MYSQL_SYSVAR_INT(sweeper_priority "Determines the priority of the background sweeper process, 0 = low (default), 1 = normal (same as user threads), 2 = high.", NULL, NULL, XT_PRIORITY_LOW, XT_PRIORITY_LOW, XT_PRIORITY_HIGH, 1); +#ifdef DRIZZLED +static MYSQL_SYSVAR_INT(max_threads, pbxt_max_threads, + PLUGIN_VAR_OPCMDARG, + "The maximum number of threads used by PBXT", + NULL, NULL, 500, 20, 20000, 1); +#else +static MYSQL_SYSVAR_INT(max_threads, pbxt_max_threads, + PLUGIN_VAR_OPCMDARG, + "The maximum number of threads used by PBXT, 0 = set according to MySQL max_connections.", + NULL, NULL, 0, 0, 20000, 1); +#endif + static struct st_mysql_sys_var* pbxt_system_variables[] = { MYSQL_SYSVAR(index_cache_size), MYSQL_SYSVAR(record_cache_size), @@ -5231,6 +5440,7 @@ static struct st_mysql_sys_var* pbxt_sys MYSQL_SYSVAR(auto_increment_mode), MYSQL_SYSVAR(offline_log_function), MYSQL_SYSVAR(sweeper_priority), + MYSQL_SYSVAR(max_threads), NULL }; #endif @@ -5241,8 +5451,8 @@ drizzle_declare_plugin(pbxt) mysql_declare_plugin(pbxt) #endif { - MYSQL_STORAGE_ENGINE_PLUGIN, #ifndef DRIZZLED + MYSQL_STORAGE_ENGINE_PLUGIN, &pbxt_storage_engine, #endif "PBXT", @@ -5266,8 +5476,8 @@ mysql_declare_plugin(pbxt) NULL /* config options */ }, { - MYSQL_INFORMATION_SCHEMA_PLUGIN, #ifndef DRIZZLED + MYSQL_INFORMATION_SCHEMA_PLUGIN, &pbxt_statitics, #endif "PBXT_STATISTICS", === modified file 'storage/pbxt/src/ha_pbxt.h' --- a/storage/pbxt/src/ha_pbxt.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/ha_pbxt.h 2009-08-17 11:12:36 +0000 @@ -28,7 +28,7 @@ #ifdef DRIZZLED #include <drizzled/common.h> #include <drizzled/handler.h> -#include <drizzled/handlerton.h> +#include <drizzled/plugin/storage_engine.h> #include <mysys/thr_lock.h> #else #include "mysql_priv.h" @@ -51,6 +51,25 @@ class ha_pbxt; +#ifdef DRIZZLED + +class PBXTStorageEngine : public StorageEngine { +public: + PBXTStorageEngine(std::string name_arg) + : StorageEngine(name_arg, HTON_NO_FLAGS) {} + + /* override */ int close_connection(Session *); + /* override */ int commit(Session *, bool); + /* override */ int rollback(Session *, bool); + /* override */ handler *create(TABLE_SHARE *, MEM_ROOT *); + /* override */ void drop_database(char *); + /* override */ bool show_status(Session *, stat_print_fn *, enum ha_stat_type); +}; + +typedef PBXTStorageEngine handlerton; + +#endif + extern handlerton *pbxt_hton; /* === modified file 'storage/pbxt/src/ha_xtsys.cc' --- a/storage/pbxt/src/ha_xtsys.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/ha_xtsys.cc 2009-08-17 11:12:36 +0000 @@ -75,7 +75,7 @@ const char **ha_xtsys::bas_ext() const return ha_pbms_exts; } -int ha_xtsys::open(const char *table_path, int mode __attribute__((unused)), uint test_if_locked __attribute__((unused))) +int ha_xtsys::open(const char *table_path, int XT_UNUSED(mode), uint XT_UNUSED(test_if_locked)) { THD *thd = current_thd; XTExceptionRec e; @@ -141,7 +141,7 @@ int ha_xtsys::close(void) return err; } -int ha_xtsys::rnd_init(bool scan __attribute__((unused))) +int ha_xtsys::rnd_init(bool XT_UNUSED(scan)) { int err = 0; @@ -185,7 +185,7 @@ int ha_xtsys::rnd_pos(byte * buf, byte * return err; } -int ha_xtsys::info(uint flag __attribute__((unused))) +int ha_xtsys::info(uint XT_UNUSED(flag)) { return 0; } @@ -211,7 +211,7 @@ int ha_xtsys::external_lock(THD *thd, in return err; } -THR_LOCK_DATA **ha_xtsys::store_lock(THD *thd __attribute__((unused)), THR_LOCK_DATA **to, enum thr_lock_type lock_type) +THR_LOCK_DATA **ha_xtsys::store_lock(THD *XT_UNUSED(thd), THR_LOCK_DATA **to, enum thr_lock_type lock_type) { if (lock_type != TL_IGNORE && ha_lock.type == TL_UNLOCK) ha_lock.type = lock_type; @@ -220,13 +220,13 @@ THR_LOCK_DATA **ha_xtsys::store_lock(THD } /* Note: ha_pbxt::delete_system_table is called instead. */ -int ha_xtsys::delete_table(const char *table_path __attribute__((unused))) +int ha_xtsys::delete_table(const char *XT_UNUSED(table_path)) { /* Should never be called */ return 0; } -int ha_xtsys::create(const char *name __attribute__((unused)), TABLE *table_arg __attribute__((unused)), HA_CREATE_INFO *create_info __attribute__((unused))) +int ha_xtsys::create(const char *XT_UNUSED(name), TABLE *XT_UNUSED(table_arg), HA_CREATE_INFO *XT_UNUSED(create_info)) { /* Allow the table to be created. * This is required after a dump is restored. @@ -234,7 +234,7 @@ int ha_xtsys::create(const char *name __ return 0; } -bool ha_xtsys::get_error_message(int error __attribute__((unused)), String *buf) +bool ha_xtsys::get_error_message(int XT_UNUSED(error), String *buf) { THD *thd = current_thd; XTExceptionRec e; === modified file 'storage/pbxt/src/ha_xtsys.h' --- a/storage/pbxt/src/ha_xtsys.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/ha_xtsys.h 2009-08-17 11:12:36 +0000 @@ -59,7 +59,7 @@ public: const char *table_type() const { return "PBXT"; } - const char *index_type(uint inx __attribute__((unused))) { + const char *index_type(uint XT_UNUSED(inx)) { return "NONE"; } @@ -69,7 +69,7 @@ public: return HA_BINLOG_ROW_CAPABLE | HA_BINLOG_STMT_CAPABLE; } - MX_ULONG_T index_flags(uint inx __attribute__((unused)), uint part __attribute__((unused)), bool all_parts __attribute__((unused))) const { + MX_ULONG_T index_flags(uint XT_UNUSED(inx), uint XT_UNUSED(part), bool XT_UNUSED(all_parts)) const { return (HA_READ_NEXT | HA_READ_PREV | HA_READ_RANGE | HA_KEYREAD_ONLY); } uint max_supported_keys() const { return 512; } === modified file 'storage/pbxt/src/hashtab_xt.cc' --- a/storage/pbxt/src/hashtab_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/hashtab_xt.cc 2009-08-17 11:12:36 +0000 @@ -115,7 +115,7 @@ xtPublic void xt_ht_put(XTThreadPtr self popr_(); } -xtPublic void *xt_ht_get(XTThreadPtr self __attribute__((unused)), XTHashTabPtr ht, void *key) +xtPublic void *xt_ht_get(XTThreadPtr XT_UNUSED(self), XTHashTabPtr ht, void *key) { XTHashItemPtr item; xtHashValue h; @@ -239,14 +239,14 @@ xtPublic void xt_ht_signal(XTThreadPtr s xt_signal_cond(self, ht->ht_cond); } -xtPublic void xt_ht_enum(struct XTThread *self __attribute__((unused)), XTHashTabPtr ht, XTHashEnumPtr en) +xtPublic void xt_ht_enum(struct XTThread *XT_UNUSED(self), XTHashTabPtr ht, XTHashEnumPtr en) { en->he_i = 0; en->he_item = NULL; en->he_ht = ht; } -xtPublic void *xt_ht_next(struct XTThread *self __attribute__((unused)), XTHashEnumPtr en) +xtPublic void *xt_ht_next(struct XTThread *XT_UNUSED(self), XTHashEnumPtr en) { if (en->he_item) { en->he_item = en->he_item->hi_next; === modified file 'storage/pbxt/src/heap_xt.cc' --- a/storage/pbxt/src/heap_xt.cc 2009-04-02 20:36:52 +0000 +++ b/storage/pbxt/src/heap_xt.cc 2009-08-17 11:12:36 +0000 @@ -31,7 +31,7 @@ #undef xt_heap_new #endif -#ifdef DEBUG +#ifdef DEBUG_MEMORY xtPublic XTHeapPtr xt_mm_heap_new(XTThreadPtr self, size_t size, XTFinalizeFunc finalize, u_int line, c_char *file, xtBool track) #else xtPublic XTHeapPtr xt_heap_new(XTThreadPtr self, size_t size, XTFinalizeFunc finalize) @@ -39,7 +39,7 @@ xtPublic XTHeapPtr xt_heap_new(XTThreadP { volatile XTHeapPtr hp; -#ifdef DEBUG +#ifdef DEBUG_MEMORY hp = (XTHeapPtr) xt_mm_calloc(self, size, line, file); hp->h_track = track; if (track) @@ -65,21 +65,21 @@ xtPublic XTHeapPtr xt_heap_new(XTThreadP return hp; } -xtPublic void xt_check_heap(XTThreadPtr self __attribute__((unused)), XTHeapPtr hp __attribute__((unused))) +xtPublic void xt_check_heap(XTThreadPtr XT_NDEBUG_UNUSED(self), XTHeapPtr XT_NDEBUG_UNUSED(hp)) { -#ifdef DEBUG +#ifdef DEBUG_MEMORY xt_mm_malloc_size(self, hp); #endif } -#ifdef DEBUG +#ifdef DEBUG_MEMORY xtPublic void xt_mm_heap_reference(XTThreadPtr self, XTHeapPtr hp, u_int line, c_char *file) #else xtPublic void xt_heap_reference(XTThreadPtr, XTHeapPtr hp) #endif { xt_spinlock_lock(&hp->h_lock); -#ifdef DEBUG +#ifdef DEBUG_MEMORY if (hp->h_track) printf("HEAP: +1 %d->%d %s:%d\n", (int) hp->h_ref_count, (int) hp->h_ref_count+1, file, (int) line); #endif @@ -91,7 +91,7 @@ xtPublic void xt_heap_release(XTThreadPt { if (!hp) return; -#ifdef DEBUG +#ifdef DEBUG_MEMORY xt_spinlock_lock(&hp->h_lock); ASSERT(hp->h_ref_count != 0); xt_spinlock_unlock(&hp->h_lock); @@ -100,7 +100,7 @@ xtPublic void xt_heap_release(XTThreadPt if (hp->h_onrelease) (*hp->h_onrelease)(self, hp); if (hp->h_ref_count > 0) { -#ifdef DEBUG +#ifdef DEBUG_MEMORY if (hp->h_track) printf("HEAP: -1 %d->%d\n", (int) hp->h_ref_count, (int) hp->h_ref_count-1); #endif @@ -116,12 +116,12 @@ xtPublic void xt_heap_release(XTThreadPt xt_spinlock_unlock(&hp->h_lock); } -xtPublic void xt_heap_set_release_callback(XTThreadPtr self __attribute__((unused)), XTHeapPtr hp, XTFinalizeFunc onrelease) +xtPublic void xt_heap_set_release_callback(XTThreadPtr XT_UNUSED(self), XTHeapPtr hp, XTFinalizeFunc onrelease) { hp->h_onrelease = onrelease; } -xtPublic u_int xt_heap_get_ref_count(struct XTThread *self __attribute__((unused)), XTHeapPtr hp) +xtPublic u_int xt_heap_get_ref_count(struct XTThread *XT_UNUSED(self), XTHeapPtr hp) { return hp->h_ref_count; } === modified file 'storage/pbxt/src/heap_xt.h' --- a/storage/pbxt/src/heap_xt.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/heap_xt.h 2009-08-17 11:12:36 +0000 @@ -25,6 +25,7 @@ #include "xt_defs.h" #include "lock_xt.h" +#include "memory_xt.h" struct XTThread; @@ -59,7 +60,7 @@ u_int xt_heap_get_ref_count(struct XTTh void xt_check_heap(struct XTThread *self, XTHeapPtr mem); -#ifdef DEBUG +#ifdef DEBUG_MEMORY #define xt_heap_new(t, s, f) xt_mm_heap_new(t, s, f, __LINE__, __FILE__, FALSE) #define xt_heap_new_track(t, s, f) xt_mm_heap_new(t, s, f, __LINE__, __FILE__, TRUE) #define xt_heap_reference(t, s) xt_mm_heap_reference(t, s, __LINE__, __FILE__) === modified file 'storage/pbxt/src/index_xt.cc' --- a/storage/pbxt/src/index_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/index_xt.cc 2009-08-18 07:46:53 +0000 @@ -23,6 +23,10 @@ #include "xt_config.h" +#ifdef DRIZZLED +#include <bitset> +#endif + #include <string.h> #include <stdio.h> #include <stddef.h> @@ -52,7 +56,7 @@ //#define CHECK_AND_PRINT //#define CHECK_NODE_REFERENCE //#define TRACE_FLUSH -//#define CHECK_PRINTS_RECORD_REFERENCES +#define CHECK_PRINTS_RECORD_REFERENCES #else #define MAX_SEARCH_DEPTH 100 #endif @@ -77,6 +81,7 @@ static u_int idx_check_index(XTOpenTable #endif static xtBool idx_insert_node(XTOpenTablePtr ot, XTIndexPtr ind, IdxBranchStackPtr stack, XTIdxKeyValuePtr key_value, xtIndexNodeID branch); +static xtBool idx_remove_lazy_deleted_item_in_node(XTOpenTablePtr ot, XTIndexPtr ind, xtIndexNodeID current, XTIndReferencePtr iref, XTIdxKeyValuePtr key_value); #ifdef XT_TRACK_INDEX_UPDATES @@ -163,7 +168,7 @@ static void track_dump_all(u_int max_blo #endif -xtPublic void xt_ind_track_dump_block(XTTableHPtr tab __attribute__((unused)), xtIndexNodeID address __attribute__((unused))) +xtPublic void xt_ind_track_dump_block(XTTableHPtr XT_UNUSED(tab), xtIndexNodeID XT_UNUSED(address)) { #ifdef TRACK_ACTIVITY u_int i = XT_NODE_ID(address)-1; @@ -268,7 +273,7 @@ static xtBool idx_new_branch(XTOpenTable if ((XT_NODE_ID(wrote_pos) = XT_NODE_ID(tab->tab_ind_free))) { /* Use the block on the free list: */ - if (!xt_ind_read_bytes(ot, wrote_pos, sizeof(XTIndFreeBlockRec), (xtWord1 *) &free_block)) + if (!xt_ind_read_bytes(ot, ind, wrote_pos, sizeof(XTIndFreeBlockRec), (xtWord1 *) &free_block)) goto failed; XT_NODE_ID(tab->tab_ind_free) = (xtIndexNodeID) XT_GET_DISK_8(free_block.if_next_block_8); xt_unlock_mutex_ns(&tab->tab_ind_lock); @@ -343,7 +348,7 @@ static xtBool idx_free_branch(XTOpenTabl * Simple compare functions */ -xtPublic int xt_compare_2_int4(XTIndexPtr ind __attribute__((unused)), uint key_length, xtWord1 *key_value, xtWord1 *b_value) +xtPublic int xt_compare_2_int4(XTIndexPtr XT_UNUSED(ind), uint key_length, xtWord1 *key_value, xtWord1 *b_value) { int r; @@ -357,7 +362,7 @@ xtPublic int xt_compare_2_int4(XTIndexPt return r; } -xtPublic int xt_compare_3_int4(XTIndexPtr ind __attribute__((unused)), uint key_length, xtWord1 *key_value, xtWord1 *b_value) +xtPublic int xt_compare_3_int4(XTIndexPtr XT_UNUSED(ind), uint key_length, xtWord1 *key_value, xtWord1 *b_value) { int r; @@ -381,7 +386,7 @@ xtPublic int xt_compare_3_int4(XTIndexPt * Tree branch sanning (searching nodes and leaves) */ -xtPublic void xt_scan_branch_single(struct XTTable *tab __attribute__((unused)), XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxKeyValuePtr value, register XTIdxResultRec *result) +xtPublic void xt_scan_branch_single(struct XTTable *XT_UNUSED(tab), XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxKeyValuePtr value, register XTIdxResultRec *result) { XT_NODE_TEMP; u_int branch_size; @@ -522,7 +527,7 @@ xtPublic void xt_scan_branch_single(stru * index (in the case of -1) or to the first value after the * the search key in the case of 1. */ -xtPublic void xt_scan_branch_fix(struct XTTable *tab __attribute__((unused)), XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxKeyValuePtr value, register XTIdxResultRec *result) +xtPublic void xt_scan_branch_fix(struct XTTable *XT_UNUSED(tab), XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxKeyValuePtr value, register XTIdxResultRec *result) { XT_NODE_TEMP; u_int branch_size; @@ -619,7 +624,7 @@ xtPublic void xt_scan_branch_fix(struct result->sr_item.i_item_offset = node_ref_size + i * full_item_size; } -xtPublic void xt_scan_branch_fix_simple(struct XTTable *tab __attribute__((unused)), XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxKeyValuePtr value, register XTIdxResultRec *result) +xtPublic void xt_scan_branch_fix_simple(struct XTTable *XT_UNUSED(tab), XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxKeyValuePtr value, register XTIdxResultRec *result) { XT_NODE_TEMP; u_int branch_size; @@ -720,7 +725,7 @@ xtPublic void xt_scan_branch_fix_simple( * Variable length key values are stored as a sorted list. Since each list item has a variable length, we * must scan the list sequentially in order to find a key. */ -xtPublic void xt_scan_branch_var(struct XTTable *tab __attribute__((unused)), XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxKeyValuePtr value, register XTIdxResultRec *result) +xtPublic void xt_scan_branch_var(struct XTTable *XT_UNUSED(tab), XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxKeyValuePtr value, register XTIdxResultRec *result) { XT_NODE_TEMP; u_int branch_size; @@ -816,7 +821,7 @@ xtPublic void xt_scan_branch_var(struct } /* Go to the next item in the node. */ -static void idx_next_branch_item(XTTableHPtr tab __attribute__((unused)), XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxResultRec *result) +static void idx_next_branch_item(XTTableHPtr XT_UNUSED(tab), XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxResultRec *result) { XT_NODE_TEMP; xtWord1 *bitem; @@ -834,7 +839,7 @@ static void idx_next_branch_item(XTTable result->sr_branch = IDX_GET_NODE_REF(tab, bitem, result->sr_item.i_node_ref_size); } -xtPublic void xt_prev_branch_item_fix(XTTableHPtr tab __attribute__((unused)), XTIndexPtr ind __attribute__((unused)), XTIdxBranchDPtr branch, register XTIdxResultRec *result) +xtPublic void xt_prev_branch_item_fix(XTTableHPtr XT_UNUSED(tab), XTIndexPtr XT_UNUSED(ind), XTIdxBranchDPtr branch, register XTIdxResultRec *result) { XT_NODE_TEMP; ASSERT_NS(result->sr_item.i_item_offset >= result->sr_item.i_item_size + result->sr_item.i_node_ref_size + result->sr_item.i_node_ref_size); @@ -843,7 +848,7 @@ xtPublic void xt_prev_branch_item_fix(XT result->sr_branch = IDX_GET_NODE_REF(tab, branch->tb_data + result->sr_item.i_item_offset, result->sr_item.i_node_ref_size); } -xtPublic void xt_prev_branch_item_var(XTTableHPtr tab __attribute__((unused)), XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxResultRec *result) +xtPublic void xt_prev_branch_item_var(XTTableHPtr XT_UNUSED(tab), XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxResultRec *result) { XT_NODE_TEMP; xtWord1 *bitem; @@ -865,7 +870,20 @@ xtPublic void xt_prev_branch_item_var(XT result->sr_item.i_item_offset = bitem - branch->tb_data; } -static void idx_first_branch_item(XTTableHPtr tab __attribute__((unused)), XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxResultPtr result) +static void idx_reload_item_fix(XTIndexPtr XT_NDEBUG_UNUSED(ind), XTIdxBranchDPtr branch, register XTIdxResultPtr result) +{ + u_int branch_size; + + branch_size = XT_GET_DISK_2(branch->tb_size_2); + ASSERT_NS(result->sr_item.i_node_ref_size == (XT_IS_NODE(branch_size) ? XT_NODE_REF_SIZE : 0)); + ASSERT_NS(result->sr_item.i_item_size == ind->mi_key_size + XT_RECORD_REF_SIZE); + result->sr_item.i_total_size = XT_GET_BRANCH_DATA_SIZE(branch_size); + if (result->sr_item.i_item_offset > result->sr_item.i_total_size) + result->sr_item.i_item_offset = result->sr_item.i_total_size; + xt_get_res_record_ref(&branch->tb_data[result->sr_item.i_item_offset + result->sr_item.i_item_size - XT_RECORD_REF_SIZE], result); +} + +static void idx_first_branch_item(XTTableHPtr XT_UNUSED(tab), XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxResultPtr result) { XT_NODE_TEMP; u_int branch_size; @@ -903,7 +921,7 @@ static void idx_first_branch_item(XTTabl /* * Last means different things for leaf or node! */ -xtPublic void xt_last_branch_item_fix(XTTableHPtr tab __attribute__((unused)), XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxResultPtr result) +xtPublic void xt_last_branch_item_fix(XTTableHPtr XT_UNUSED(tab), XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxResultPtr result) { XT_NODE_TEMP; u_int branch_size; @@ -935,7 +953,7 @@ xtPublic void xt_last_branch_item_fix(XT } } -xtPublic void xt_last_branch_item_var(XTTableHPtr tab __attribute__((unused)), XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxResultPtr result) +xtPublic void xt_last_branch_item_var(XTTableHPtr XT_UNUSED(tab), XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxResultPtr result) { XT_NODE_TEMP; u_int branch_size; @@ -986,6 +1004,218 @@ xtPublic void xt_last_branch_item_var(XT } } +xtPublic xtBool xt_idx_lazy_delete_on_leaf(XTIndexPtr ind, XTIndBlockPtr block, xtWord2 branch_size) +{ + ASSERT_NS(ind->mi_fix_key); + + /* Compact the leaf if more than half the items that fit on the page + * are deleted: */ + if (block->cp_del_count >= ind->mi_max_items/2) + return FALSE; + + /* Compact the page if there is only 1 (or less) valid item left: */ + if ((u_int) block->cp_del_count+1 >= ((u_int) branch_size - 2)/(ind->mi_key_size + XT_RECORD_REF_SIZE)) + return FALSE; + + return OK; +} + +static xtBool idx_lazy_delete_on_node(XTIndexPtr ind, XTIndBlockPtr block, register XTIdxItemPtr item) +{ + ASSERT_NS(ind->mi_fix_key); + + /* Compact the node if more than 1/4 of the items that fit on the page + * are deleted: */ + if (block->cp_del_count >= ind->mi_max_items/4) + return FALSE; + + /* Compact the page if there is only 1 (or less) valid item left: */ + if ((u_int) block->cp_del_count+1 >= (item->i_total_size - item->i_node_ref_size)/(item->i_item_size + item->i_node_ref_size)) + return FALSE; + + return OK; +} + +inline static xtBool idx_cmp_item_key_fix(XTIndReferencePtr iref, register XTIdxItemPtr item, XTIdxKeyValuePtr value) +{ + xtWord1 *data; + + data = &iref->ir_branch->tb_data[item->i_item_offset]; + return memcmp(data, value->sv_key, value->sv_length) == 0; +} + +inline static void idx_set_item_key_fix(XTIndReferencePtr iref, register XTIdxItemPtr item, XTIdxKeyValuePtr value) +{ + xtWord1 *data; + + data = &iref->ir_branch->tb_data[item->i_item_offset]; + memcpy(data, value->sv_key, value->sv_length); + xt_set_val_record_ref(data + value->sv_length, value); + iref->ir_updated = TRUE; +} + +inline static void idx_set_item_reference(XTIndReferencePtr iref, register XTIdxItemPtr item, xtRowID rec_id, xtRowID row_id) +{ + size_t offset; + xtWord1 *data; + + /* This is the offset of the reference in the item we found: */ + offset = item->i_item_offset +item->i_item_size - XT_RECORD_REF_SIZE; + data = &iref->ir_branch->tb_data[offset]; + + xt_set_record_ref(data, rec_id, row_id); + iref->ir_updated = TRUE; +} + +inline static void idx_set_item_row_id(XTIndReferencePtr iref, register XTIdxItemPtr item, xtRowID row_id) +{ + size_t offset; + xtWord1 *data; + + offset = + /* This is the offset of the reference in the item we found: */ + item->i_item_offset +item->i_item_size - XT_RECORD_REF_SIZE + + /* This is the offset of the row id in the reference: */ + XT_RECORD_ID_SIZE; + data = &iref->ir_branch->tb_data[offset]; + + /* This update does not change the structure of page, so we do it without + * copying the page before we write. + */ + XT_SET_DISK_4(data, row_id); + iref->ir_updated = TRUE; +} + +inline static xtBool idx_is_item_deleted(register XTIdxBranchDPtr branch, register XTIdxItemPtr item) +{ + xtWord1 *data; + + data = &branch->tb_data[item->i_item_offset + item->i_item_size - XT_RECORD_REF_SIZE + XT_RECORD_ID_SIZE]; + return XT_GET_DISK_4(data) == (xtRowID) -1; +} + +inline static void idx_set_item_deleted(XTIndReferencePtr iref, register XTIdxItemPtr item) +{ + idx_set_item_row_id(iref, item, (xtRowID) -1); + + /* This should be safe because there is only one thread, + * the sweeper, that does this! + * + * Threads that decrement this value have an xlock on + * the page, or the index. + */ + iref->ir_block->cp_del_count++; +} + +/* + * {LAZY-DEL-INDEX-ITEMS} + * Do a lazy delete of an item by just setting the Row ID + * to the delete indicator: row ID -1. + */ +static void idx_lazy_delete_branch_item(XTOpenTablePtr ot, XTIndexPtr ind, XTIndReferencePtr iref, register XTIdxItemPtr item) +{ + idx_set_item_deleted(iref, item); + xt_ind_release(ot, ind, iref->ir_xlock ? XT_UNLOCK_W_UPDATE : XT_UNLOCK_R_UPDATE, iref); +} + +/* + * This function compacts the leaf, but preserves the + * position of the item. + */ +static xtBool idx_compact_leaf(XTOpenTablePtr ot, XTIndexPtr ind, XTIndReferencePtr iref, register XTIdxItemPtr item) +{ + register XTIdxBranchDPtr branch = iref->ir_branch; + int item_idx, count, i, idx; + u_int size; + xtWord1 *s_data; + xtWord1 *d_data; + xtWord1 *data; + xtRowID row_id; + + if (iref->ir_block->cb_handle_count) { + if (!xt_ind_copy_on_write(iref)) { + xt_ind_release(ot, ind, iref->ir_xlock ? XT_UNLOCK_WRITE : XT_UNLOCK_READ, iref); + return FAILED; + } + } + + ASSERT_NS(!item->i_node_ref_size); + ASSERT_NS(ind->mi_fix_key); + size = item->i_item_size; + count = item->i_total_size / size; + item_idx = item->i_item_offset / size; + s_data = d_data = branch->tb_data; + idx = 0; + for (i=0; i<count; i++) { + data = s_data + item->i_item_size - XT_RECORD_REF_SIZE + XT_RECORD_ID_SIZE; + row_id = XT_GET_DISK_4(data); + if (row_id == (xtRowID) -1) { + if (idx < item_idx) + item_idx--; + } + else { + if (d_data != s_data) + memcpy(d_data, s_data, size); + d_data += size; + idx++; + } + s_data += size; + } + iref->ir_block->cp_del_count = 0; + item->i_total_size = d_data - branch->tb_data; + ASSERT_NS(idx * size == item->i_total_size); + item->i_item_offset = item_idx * size; + XT_SET_DISK_2(branch->tb_size_2, XT_MAKE_BRANCH_SIZE(item->i_total_size, 0)); + iref->ir_updated = TRUE; + return OK; +} + +static xtBool idx_lazy_remove_leaf_item_right(XTOpenTablePtr ot, XTIndexPtr ind, XTIndReferencePtr iref, register XTIdxItemPtr item) +{ + register XTIdxBranchDPtr branch = iref->ir_branch; + int item_idx, count, i; + u_int size; + xtWord1 *s_data; + xtWord1 *d_data; + xtWord1 *data; + xtRowID row_id; + + ASSERT_NS(!item->i_node_ref_size); + + if (iref->ir_block->cb_handle_count) { + if (!xt_ind_copy_on_write(iref)) { + xt_ind_release(ot, ind, XT_UNLOCK_WRITE, iref); + return FAILED; + } + } + + ASSERT_NS(ind->mi_fix_key); + size = item->i_item_size; + count = item->i_total_size / size; + item_idx = item->i_item_offset / size; + s_data = d_data = branch->tb_data; + for (i=0; i<count; i++) { + if (i == item_idx) + item->i_item_offset = d_data - branch->tb_data; + else { + data = s_data + item->i_item_size - XT_RECORD_REF_SIZE + XT_RECORD_ID_SIZE; + row_id = XT_GET_DISK_4(data); + if (row_id != (xtRowID) -1) { + if (d_data != s_data) + memcpy(d_data, s_data, size); + d_data += size; + } + } + s_data += size; + } + iref->ir_block->cp_del_count = 0; + item->i_total_size = d_data - branch->tb_data; + XT_SET_DISK_2(branch->tb_size_2, XT_MAKE_BRANCH_SIZE(item->i_total_size, 0)); + iref->ir_updated = TRUE; + xt_ind_release(ot, ind, XT_UNLOCK_W_UPDATE, iref); + return OK; +} + /* * Remove an item and save to disk. */ @@ -1003,8 +1233,14 @@ static xtBool idx_remove_branch_item_rig * an Xlock on the cache block. */ if (iref->ir_block->cb_handle_count) { - if (!xt_ind_copy_on_write(iref)) + if (!xt_ind_copy_on_write(iref)) { + xt_ind_release(ot, ind, item->i_node_ref_size ? XT_UNLOCK_READ : XT_UNLOCK_WRITE, iref); return FAILED; + } + } + if (ind->mi_lazy_delete) { + if (idx_is_item_deleted(branch, item)) + iref->ir_block->cp_del_count--; } /* Remove the node reference to the left of the item: */ memmove(&branch->tb_data[item->i_item_offset], @@ -1013,18 +1249,28 @@ static xtBool idx_remove_branch_item_rig item->i_total_size -= size; XT_SET_DISK_2(branch->tb_size_2, XT_MAKE_BRANCH_SIZE(item->i_total_size, item->i_node_ref_size)); IDX_TRACE("%d-> %x\n", (int) XT_NODE_ID(address), (int) XT_GET_DISK_2(branch->tb_size_2)); + iref->ir_updated = TRUE; xt_ind_release(ot, ind, item->i_node_ref_size ? XT_UNLOCK_R_UPDATE : XT_UNLOCK_W_UPDATE, iref); return OK; } -static xtBool idx_remove_branch_item_left(XTOpenTablePtr ot, XTIndexPtr ind, xtIndexNodeID, XTIndReferencePtr iref, register XTIdxItemPtr item) +static xtBool idx_remove_branch_item_left(XTOpenTablePtr ot, XTIndexPtr ind, xtIndexNodeID, XTIndReferencePtr iref, register XTIdxItemPtr item, xtBool *lazy_delete_cleanup_required) { register XTIdxBranchDPtr branch = iref->ir_branch; u_int size = item->i_item_size + item->i_node_ref_size; + ASSERT_NS(item->i_node_ref_size); if (iref->ir_block->cb_handle_count) { - if (!xt_ind_copy_on_write(iref)) + if (!xt_ind_copy_on_write(iref)) { + xt_ind_release(ot, ind, item->i_node_ref_size ? XT_UNLOCK_READ : XT_UNLOCK_WRITE, iref); return FAILED; + } + } + if (ind->mi_lazy_delete) { + if (idx_is_item_deleted(branch, item)) + iref->ir_block->cp_del_count--; + if (lazy_delete_cleanup_required) + *lazy_delete_cleanup_required = idx_lazy_delete_on_node(ind, iref->ir_block, item); } /* Remove the node reference to the left of the item: */ memmove(&branch->tb_data[item->i_item_offset - item->i_node_ref_size], @@ -1033,11 +1279,12 @@ static xtBool idx_remove_branch_item_lef item->i_total_size -= size; XT_SET_DISK_2(branch->tb_size_2, XT_MAKE_BRANCH_SIZE(item->i_total_size, item->i_node_ref_size)); IDX_TRACE("%d-> %x\n", (int) XT_NODE_ID(address), (int) XT_GET_DISK_2(branch->tb_size_2)); + iref->ir_updated = TRUE; xt_ind_release(ot, ind, item->i_node_ref_size ? XT_UNLOCK_R_UPDATE : XT_UNLOCK_W_UPDATE, iref); return OK; } -static void idx_insert_leaf_item(XTIndexPtr ind __attribute__((unused)), XTIdxBranchDPtr leaf, XTIdxKeyValuePtr value, XTIdxResultPtr result) +static void idx_insert_leaf_item(XTIndexPtr XT_UNUSED(ind), XTIdxBranchDPtr leaf, XTIdxKeyValuePtr value, XTIdxResultPtr result) { xtWord1 *item; @@ -1053,7 +1300,7 @@ static void idx_insert_leaf_item(XTIndex XT_SET_DISK_2(leaf->tb_size_2, XT_MAKE_LEAF_SIZE(result->sr_item.i_total_size)); } -static void idx_insert_node_item(XTTableHPtr tab __attribute__((unused)), XTIndexPtr ind __attribute__((unused)), XTIdxBranchDPtr leaf, XTIdxKeyValuePtr value, XTIdxResultPtr result, xtIndexNodeID branch) +static void idx_insert_node_item(XTTableHPtr XT_UNUSED(tab), XTIndexPtr XT_UNUSED(ind), XTIdxBranchDPtr leaf, XTIdxKeyValuePtr value, XTIdxResultPtr result, xtIndexNodeID branch) { xtWord1 *item; @@ -1114,7 +1361,7 @@ static void idx_get_middle_branch_item(X } } -static size_t idx_write_branch_item(XTIndexPtr ind __attribute__((unused)), xtWord1 *item, XTIdxKeyValuePtr value) +static size_t idx_write_branch_item(XTIndexPtr XT_UNUSED(ind), xtWord1 *item, XTIdxKeyValuePtr value) { memcpy(item, value->sv_key, value->sv_length); xt_set_val_record_ref(item + value->sv_length, value); @@ -1133,23 +1380,38 @@ static xtBool idx_replace_node_key(XTOpe xtWord1 key_buf[XT_INDEX_MAX_KEY_SIZE]; #ifdef DEBUG - iref.ir_ulock = XT_UNLOCK_NONE; + iref.ir_xlock = 2; + iref.ir_updated = 2; #endif - if (!xt_ind_fetch(ot, current, XT_LOCK_WRITE, &iref)) + if (!xt_ind_fetch(ot, ind, current, XT_LOCK_WRITE, &iref)) return FAILED; if (iref.ir_block->cb_handle_count) { if (!xt_ind_copy_on_write(&iref)) goto failed_1; } + if (ind->mi_lazy_delete) { + ASSERT_NS(item_size == item->i_pos.i_item_size); + if (idx_is_item_deleted(iref.ir_branch, &item->i_pos)) + iref.ir_block->cp_del_count--; + } memmove(&iref.ir_branch->tb_data[item->i_pos.i_item_offset + item_size], &iref.ir_branch->tb_data[item->i_pos.i_item_offset + item->i_pos.i_item_size], item->i_pos.i_total_size - item->i_pos.i_item_offset - item->i_pos.i_item_size); memcpy(&iref.ir_branch->tb_data[item->i_pos.i_item_offset], item_buf, item_size); + if (ind->mi_lazy_delete) { + if (idx_is_item_deleted(iref.ir_branch, &item->i_pos)) + iref.ir_block->cp_del_count++; + } item->i_pos.i_total_size = item->i_pos.i_total_size + item_size - item->i_pos.i_item_size; XT_SET_DISK_2(iref.ir_branch->tb_size_2, XT_MAKE_NODE_SIZE(item->i_pos.i_total_size)); IDX_TRACE("%d-> %x\n", (int) XT_NODE_ID(current), (int) XT_GET_DISK_2(iref.ir_branch->tb_size_2)); + iref.ir_updated = TRUE; +#ifdef DEBUG + if (ind->mi_lazy_delete) + ASSERT_NS(item->i_pos.i_total_size <= XT_INDEX_PAGE_DATA_SIZE); +#endif if (item->i_pos.i_total_size <= XT_INDEX_PAGE_DATA_SIZE) return xt_ind_release(ot, ind, XT_UNLOCK_W_UPDATE, &iref); @@ -1184,6 +1446,7 @@ static xtBool idx_replace_node_key(XTOpe /* Change the size of the old branch: */ XT_SET_DISK_2(iref.ir_branch->tb_size_2, XT_MAKE_NODE_SIZE(result.sr_item.i_item_offset)); IDX_TRACE("%d-> %x\n", (int) XT_NODE_ID(current), (int) XT_GET_DISK_2(iref.ir_branch->tb_size_2)); + iref.ir_updated = TRUE; xt_ind_release(ot, ind, XT_UNLOCK_W_UPDATE, &iref); @@ -1237,7 +1500,8 @@ static xtBool idx_insert_node(XTOpenTabl XTIdxBranchDPtr new_branch_ptr; #ifdef DEBUG - iref.ir_ulock = XT_UNLOCK_NONE; + iref.ir_xlock = 2; + iref.ir_updated = 2; #endif /* Insert a new branch (key, data)... */ if (!(stack_item = idx_pop(stack))) { @@ -1268,7 +1532,7 @@ static xtBool idx_insert_node(XTOpenTabl * cache, and will remain in cache when we read again below for the * purpose of update. */ - if (!xt_ind_fetch(ot, current, XT_LOCK_READ, &iref)) + if (!xt_ind_fetch(ot, ind, current, XT_LOCK_READ, &iref)) goto failed; ASSERT_NS(XT_IS_NODE(XT_GET_DISK_2(iref.ir_branch->tb_size_2))); ind->mi_scan_branch(ot->ot_table, ind, iref.ir_branch, key_value, &result); @@ -1280,6 +1544,7 @@ static xtBool idx_insert_node(XTOpenTabl } idx_insert_node_item(ot->ot_table, ind, iref.ir_branch, key_value, &result, branch); IDX_TRACE("%d-> %x\n", (int) XT_NODE_ID(current), (int) XT_GET_DISK_2(ot->ot_ind_wbuf.tb_size_2)); + iref.ir_updated = TRUE; ASSERT_NS(result.sr_item.i_total_size <= XT_INDEX_PAGE_DATA_SIZE); xt_ind_release(ot, ind, XT_UNLOCK_R_UPDATE, &iref); goto done_ok; @@ -1314,6 +1579,7 @@ static xtBool idx_insert_node(XTOpenTabl goto failed_2; } memcpy(iref.ir_branch, &ot->ot_ind_wbuf, offsetof(XTIdxBranchDRec, tb_data) + result.sr_item.i_item_offset); + iref.ir_updated = TRUE; xt_ind_release(ot, ind, XT_UNLOCK_R_UPDATE, &iref); /* Insert the new branch into the parent node, using the new middle key value: */ @@ -1373,7 +1639,8 @@ static xtBool idx_check_duplicates(XTOpe XTXactWaitRec xw; #ifdef DEBUG - iref.ir_ulock = XT_UNLOCK_NONE; + iref.ir_xlock = 2; + iref.ir_updated = 2; #endif retry: idx_newstack(&stack); @@ -1385,7 +1652,7 @@ static xtBool idx_check_duplicates(XTOpe key_value->sv_flags = 0; while (XT_NODE_ID(current)) { - if (!xt_ind_fetch(ot, current, XT_LOCK_READ, &iref)) { + if (!xt_ind_fetch(ot, ind, current, XT_LOCK_READ, &iref)) { key_value->sv_flags = save_flags; return FAILED; } @@ -1422,7 +1689,7 @@ static xtBool idx_check_duplicates(XTOpe while ((node = idx_pop(&stack))) { if (node->i_pos.i_item_offset < node->i_pos.i_total_size) { current = node->i_branch; - if (!xt_ind_fetch(ot, current, XT_LOCK_READ, &iref)) + if (!xt_ind_fetch(ot, ind, current, XT_LOCK_READ, &iref)) return FAILED; xt_get_res_record_ref(&iref.ir_branch->tb_data[node->i_pos.i_item_offset + node->i_pos.i_item_size - XT_RECORD_REF_SIZE], &result); result.sr_item = node->i_pos; @@ -1439,6 +1706,11 @@ static xtBool idx_check_duplicates(XTOpe break; } + if (ind->mi_lazy_delete) { + if (result.sr_row_id == (xtRowID) -1) + goto next_item; + } + switch (xt_tab_maybe_committed(ot, result.sr_rec_id, &xn_id, NULL, NULL)) { case XT_MAYBE: /* Record is not committed, wait for the transaction. */ @@ -1464,6 +1736,7 @@ static xtBool idx_check_duplicates(XTOpe break; } + next_item: idx_next_branch_item(ot->ot_table, ind, iref.ir_branch, &result); if (result.sr_item.i_node_ref_size) { @@ -1473,7 +1746,7 @@ static xtBool idx_check_duplicates(XTOpe if (!idx_push(&stack, current, &result.sr_item)) return FAILED; current = result.sr_branch; - if (!xt_ind_fetch(ot, current, XT_LOCK_READ, &iref)) + if (!xt_ind_fetch(ot, ind, current, XT_LOCK_READ, &iref)) return FAILED; idx_first_branch_item(ot->ot_table, ind, iref.ir_branch, &result); if (!result.sr_item.i_node_ref_size) @@ -1489,6 +1762,14 @@ static xtBool idx_check_duplicates(XTOpe return FAILED; } +inline static void idx_still_on_key(XTIndexPtr ind, register XTIdxSearchKeyPtr search_key, register XTIdxBranchDPtr branch, register XTIdxItemPtr item) +{ + if (search_key && search_key->sk_on_key) { + search_key->sk_on_key = myxt_compare_key(ind, search_key->sk_key_value.sv_flags, search_key->sk_key_value.sv_length, + search_key->sk_key_value.sv_key, &branch->tb_data[item->i_item_offset]) == 0; + } +} + /* * Insert a value into the given index. Return FALSE if an error occurs. */ @@ -1506,9 +1787,11 @@ xtPublic xtBool xt_idx_insert(XTOpenTabl size_t new_size; xtBool check_for_dups = ind->mi_flags & (HA_UNIQUE_CHECK | HA_NOSAME) && !allow_dups; xtBool lock_structure = FALSE; + xtBool updated = FALSE; #ifdef DEBUG - iref.ir_ulock = XT_UNLOCK_NONE; + iref.ir_xlock = 2; + iref.ir_updated = 2; #endif #ifdef CHECK_AND_PRINT //idx_check_index(ot, ind, TRUE); @@ -1559,6 +1842,7 @@ xtPublic xtBool xt_idx_insert(XTOpenTabl XT_INDEX_READ_LOCK(ind, ot); retry: + /* Create a root node if required: */ if (!(XT_NODE_ID(current) = XT_NODE_ID(ind->mi_root))) { /* Index is empty, create a new one: */ ASSERT_NS(lock_structure); @@ -1575,8 +1859,9 @@ xtPublic xtBool xt_idx_insert(XTOpenTabl goto done_ok; } + /* Search down the tree for the insertion point. */ while (XT_NODE_ID(current)) { - if (!xt_ind_fetch(ot, current, XT_XLOCK_LEAF, &iref)) + if (!xt_ind_fetch(ot, ind, current, XT_XLOCK_LEAF, &iref)) goto failed; ind->mi_scan_branch(ot->ot_table, ind, iref.ir_branch, &key_value, &result); if (result.sr_duplicate) { @@ -1601,8 +1886,23 @@ xtPublic xtBool xt_idx_insert(XTOpenTabl } } if (result.sr_found) { - /* Node found, can happen during recovery of indexes! */ - XTPageUnlockType utype; + /* Node found, can happen during recovery of indexes! + * We have found an exact match of both key and record. + */ + XTPageUnlockType utype; + xtBool overwrite = FALSE; + + /* {LAZY-DEL-INDEX-ITEMS} + * If the item has been lazy deleted, then just overwrite! + */ + if (result.sr_row_id == (xtRowID) -1) { + xtWord2 del_count; + + /* This is safe because we have an xlock on the leaf. */ + if ((del_count = iref.ir_block->cp_del_count)) + iref.ir_block->cp_del_count = del_count-1; + overwrite = TRUE; + } if (!result.sr_row_id && row_id) { /* {INDEX-RECOV_ROWID} Set the row-id @@ -1610,20 +1910,11 @@ xtPublic xtBool xt_idx_insert(XTOpenTabl * is not committed. * It will be removed later by the sweeper. */ - size_t offset; - xtWord1 *data; - - offset = - /* This is the offset of the reference in the item we found: */ - result.sr_item.i_item_offset + result.sr_item.i_item_size - XT_RECORD_REF_SIZE + - /* This is the offset of the row id in the reference: */ - 4; - data = &iref.ir_branch->tb_data[offset]; + overwrite = TRUE; + } - /* This update does not change the structure of page, so we do it without - * copying the page before we write. - */ - XT_SET_DISK_4(data, row_id); + if (overwrite) { + idx_set_item_row_id(&iref, &result.sr_item, row_id); utype = result.sr_item.i_node_ref_size ? XT_UNLOCK_R_UPDATE : XT_UNLOCK_W_UPDATE; } else @@ -1644,14 +1935,84 @@ xtPublic xtBool xt_idx_insert(XTOpenTabl /* Must be a leaf!: */ ASSERT_NS(!result.sr_item.i_node_ref_size); + updated = FALSE; + if (ind->mi_lazy_delete && iref.ir_block->cp_del_count) { + /* There are a number of possibilities: + * - We could just replace a lazy deleted slot. + * - We could compact and insert. + * - We could just insert + */ + + if (result.sr_item.i_item_offset > 0) { + /* Check if it can go into the previous node: */ + XTIdxResultRec t_res; + + t_res.sr_item = result.sr_item; + xt_prev_branch_item_fix(ot->ot_table, ind, iref.ir_branch, &t_res); + if (t_res.sr_row_id != (xtRowID) -1) + goto try_current; + + /* Yup, it can, but first check to see if it would be + * better to put it in the current node. + * This is the case if the previous node key is not the + * same as the key we are adding... + */ + if (result.sr_item.i_item_offset < result.sr_item.i_total_size && + result.sr_row_id == (xtRowID) -1) { + if (!idx_cmp_item_key_fix(&iref, &t_res.sr_item, &key_value)) + goto try_current; + } + + idx_set_item_key_fix(&iref, &t_res.sr_item, &key_value); + iref.ir_block->cp_del_count--; + xt_ind_release(ot, ind, XT_UNLOCK_W_UPDATE, &iref); + goto done_ok; + } + + try_current: + if (result.sr_item.i_item_offset < result.sr_item.i_total_size) { + if (result.sr_row_id == (xtRowID) -1) { + idx_set_item_key_fix(&iref, &result.sr_item, &key_value); + iref.ir_block->cp_del_count--; + xt_ind_release(ot, ind, XT_UNLOCK_W_UPDATE, &iref); + goto done_ok; + } + } + + /* Check if we must compact... + * It makes no sense to split as long as there are lazy deleted items + * in the page. So, delete them if a split would otherwise be required! + */ + ASSERT_NS(key_value.sv_length + XT_RECORD_REF_SIZE == result.sr_item.i_item_size); + if (result.sr_item.i_total_size + key_value.sv_length + XT_RECORD_REF_SIZE > XT_INDEX_PAGE_DATA_SIZE) { + if (!idx_compact_leaf(ot, ind, &iref, &result.sr_item)) + goto failed; + updated = TRUE; + } + + /* Fall through to the insert code... */ + /* NOTE: if there were no lazy deleted items in the leaf, then + * idx_compact_leaf is a NOP. This is the only case in which it may not + * fall through and do the insert below. + * + * Normally, if the cp_del_count is correct then the insert + * will work below, and the assertion here will not fail. + * + * In this case, the xt_ind_release() will correctly indicate an update. + */ + ASSERT_NS(result.sr_item.i_total_size + key_value.sv_length + XT_RECORD_REF_SIZE <= XT_INDEX_PAGE_DATA_SIZE); + } + if (result.sr_item.i_total_size + key_value.sv_length + XT_RECORD_REF_SIZE <= XT_INDEX_PAGE_DATA_SIZE) { if (iref.ir_block->cb_handle_count) { if (!xt_ind_copy_on_write(&iref)) goto failed_1; } + idx_insert_leaf_item(ind, iref.ir_branch, &key_value, &result); IDX_TRACE("%d-> %x\n", (int) XT_NODE_ID(current), (int) XT_GET_DISK_2(ot->ot_ind_wbuf.tb_size_2)); ASSERT_NS(result.sr_item.i_total_size <= XT_INDEX_PAGE_DATA_SIZE); + iref.ir_updated = TRUE; xt_ind_release(ot, ind, XT_UNLOCK_W_UPDATE, &iref); goto done_ok; } @@ -1660,7 +2021,7 @@ xtPublic xtBool xt_idx_insert(XTOpenTabl * Make sure we have a structural lock: */ if (!lock_structure) { - xt_ind_release(ot, ind, XT_UNLOCK_WRITE, &iref); + xt_ind_release(ot, ind, updated ? XT_UNLOCK_W_UPDATE : XT_UNLOCK_WRITE, &iref); XT_INDEX_UNLOCK(ind, ot); lock_structure = TRUE; goto lock_and_retry; @@ -1705,6 +2066,7 @@ xtPublic xtBool xt_idx_insert(XTOpenTabl goto failed_2; } memcpy(iref.ir_branch, &ot->ot_ind_wbuf, offsetof(XTIdxBranchDRec, tb_data) + result.sr_item.i_item_offset); + iref.ir_updated = TRUE; xt_ind_release(ot, ind, XT_UNLOCK_W_UPDATE, &iref); /* Insert the new branch into the parent node, using the new middle key value: */ @@ -1732,7 +2094,7 @@ xtPublic xtBool xt_idx_insert(XTOpenTabl idx_free_branch(ot, ind, new_branch); failed_1: - xt_ind_release(ot, ind, XT_UNLOCK_WRITE, &iref); + xt_ind_release(ot, ind, updated ? XT_UNLOCK_W_UPDATE : XT_UNLOCK_WRITE, &iref); failed: XT_INDEX_UNLOCK(ind, ot); @@ -1747,18 +2109,175 @@ xtPublic xtBool xt_idx_insert(XTOpenTabl return FAILED; } + +/* Remove the given item in the node. + * This is done by going down the tree to find a replacement + * for the deleted item! + */ +static xtBool idx_remove_item_in_node(XTOpenTablePtr ot, XTIndexPtr ind, IdxBranchStackPtr stack, XTIndReferencePtr iref, XTIdxKeyValuePtr key_value) +{ + IdxStackItemPtr delete_node; + XTIdxResultRec result; + xtIndexNodeID current; + xtBool lazy_delete_cleanup_required = FALSE; + IdxStackItemPtr current_top; + + delete_node = idx_top(stack); + current = delete_node->i_branch; + result.sr_item = delete_node->i_pos; + + /* Follow the branch after this item: */ + idx_next_branch_item(ot->ot_table, ind, iref->ir_branch, &result); + xt_ind_release(ot, ind, iref->ir_updated ? XT_UNLOCK_R_UPDATE : XT_UNLOCK_READ, iref); + + /* Go down the left-hand side until we reach a leaf: */ + while (XT_NODE_ID(current)) { + current = result.sr_branch; + if (!xt_ind_fetch(ot, ind, current, XT_XLOCK_LEAF, iref)) + return FAILED; + idx_first_branch_item(ot->ot_table, ind, iref->ir_branch, &result); + if (!result.sr_item.i_node_ref_size) + break; + xt_ind_release(ot, ind, XT_UNLOCK_READ, iref); + if (!idx_push(stack, current, &result.sr_item)) + return FAILED; + } + + ASSERT_NS(XT_NODE_ID(current)); + ASSERT_NS(!result.sr_item.i_node_ref_size); + + if (!xt_ind_reserve(ot, stack->s_top + 2, iref->ir_branch)) { + xt_ind_release(ot, ind, XT_UNLOCK_WRITE, iref); + return FAILED; + } + + /* This code removes lazy deleted items from the leaf, + * before we promote an item to a leaf. + * This is not essential, but prevents lazy deleted + * items from being propogated up the tree. + */ + if (ind->mi_lazy_delete) { + if (iref->ir_block->cp_del_count) { + if (!idx_compact_leaf(ot, ind, iref, &result.sr_item)) + return FAILED; + } + } + + /* Crawl back up the stack trace, looking for a key + * that can be used to replace the deleted key. + * + * Any empty nodes on the way up can be removed! + */ + if (result.sr_item.i_total_size > 0) { + /* There is a key in the leaf, extract it, and put it in the node: */ + memcpy(key_value->sv_key, &iref->ir_branch->tb_data[result.sr_item.i_item_offset], result.sr_item.i_item_size); + /* This call also frees the iref.ir_branch page! */ + if (!idx_remove_branch_item_right(ot, ind, current, iref, &result.sr_item)) + return FAILED; + if (!idx_replace_node_key(ot, ind, delete_node, stack, result.sr_item.i_item_size, key_value->sv_key)) + return FAILED; + goto done_ok; + } + + xt_ind_release(ot, ind, iref->ir_updated ? XT_UNLOCK_W_UPDATE : XT_UNLOCK_WRITE, iref); + + for (;;) { + /* The current node/leaf is empty, remove it: */ + idx_free_branch(ot, ind, current); + + current_top = idx_pop(stack); + current = current_top->i_branch; + if (!xt_ind_fetch(ot, ind, current, XT_XLOCK_LEAF, iref)) + return FAILED; + + if (current_top == delete_node) { + /* All children have been removed. Delete the key and done: */ + if (!idx_remove_branch_item_right(ot, ind, current, iref, &current_top->i_pos)) + return FAILED; + goto done_ok; + } + + if (current_top->i_pos.i_total_size > current_top->i_pos.i_node_ref_size) { + /* Save the key: */ + memcpy(key_value->sv_key, &iref->ir_branch->tb_data[current_top->i_pos.i_item_offset], current_top->i_pos.i_item_size); + /* This function also frees the cache page: */ + if (!idx_remove_branch_item_left(ot, ind, current, iref, &current_top->i_pos, &lazy_delete_cleanup_required)) + return FAILED; + if (!idx_replace_node_key(ot, ind, delete_node, stack, current_top->i_pos.i_item_size, key_value->sv_key)) + return FAILED; + /* */ + if (lazy_delete_cleanup_required) { + if (!xt_ind_fetch(ot, ind, current, XT_LOCK_READ, iref)) + return FAILED; + if (!idx_remove_lazy_deleted_item_in_node(ot, ind, current, iref, key_value)) + return FAILED; + } + goto done_ok; + } + xt_ind_release(ot, ind, current_top->i_pos.i_node_ref_size ? XT_UNLOCK_READ : XT_UNLOCK_WRITE, iref); + } + + done_ok: +#ifdef XT_TRACK_INDEX_UPDATES + ASSERT_NS(ot->ot_ind_reserved >= ot->ot_ind_reads); +#endif + return OK; +} + +/* + * This function assumes we have a lock on the structure of the index. + */ +static xtBool idx_remove_lazy_deleted_item_in_node(XTOpenTablePtr ot, XTIndexPtr ind, xtIndexNodeID current, XTIndReferencePtr iref, XTIdxKeyValuePtr key_value) +{ + IdxBranchStackRec stack; + XTIdxResultRec result; + + /* Now remove all lazy deleted items in this node.... */ + idx_first_branch_item(ot->ot_table, ind, (XTIdxBranchDPtr) iref->ir_block->cb_data, &result); + + for (;;) { + while (result.sr_item.i_item_offset < result.sr_item.i_total_size) { + if (result.sr_row_id == (xtRowID) -1) + goto remove_item; + idx_next_branch_item(ot->ot_table, ind, (XTIdxBranchDPtr) iref->ir_block->cb_data, &result); + } + break; + + remove_item: + + idx_newstack(&stack); + if (!idx_push(&stack, current, &result.sr_item)) { + xt_ind_release(ot, ind, iref->ir_updated ? XT_UNLOCK_R_UPDATE : XT_UNLOCK_READ, iref); + return FAILED; + } + + if (!idx_remove_item_in_node(ot, ind, &stack, iref, key_value)) + return FAILED; + + /* Go back up to the node we are trying to + * free of things. + */ + if (!xt_ind_fetch(ot, ind, current, XT_LOCK_READ, iref)) + return FAILED; + /* Load the data again: */ + idx_reload_item_fix(ind, iref->ir_branch, &result); + } + + xt_ind_release(ot, ind, iref->ir_updated ? XT_UNLOCK_R_UPDATE : XT_UNLOCK_READ, iref); + return OK; +} + static xtBool idx_delete(XTOpenTablePtr ot, XTIndexPtr ind, XTIdxKeyValuePtr key_value) { IdxBranchStackRec stack; xtIndexNodeID current; XTIndReferenceRec iref; XTIdxResultRec result; - IdxStackItemPtr delete_node = NULL; - IdxStackItemPtr current_top = NULL; xtBool lock_structure = FALSE; #ifdef DEBUG - iref.ir_ulock = XT_UNLOCK_NONE; + iref.ir_xlock = 2; + iref.ir_updated = 2; #endif /* The index appears to have no root: */ if (!XT_NODE_ID(ind->mi_root)) @@ -1776,17 +2295,37 @@ static xtBool idx_delete(XTOpenTablePtr goto done_ok; while (XT_NODE_ID(current)) { - if (!xt_ind_fetch(ot, current, XT_XLOCK_LEAF, &iref)) + if (!xt_ind_fetch(ot, ind, current, XT_XLOCK_DEL_LEAF, &iref)) goto failed; ind->mi_scan_branch(ot->ot_table, ind, iref.ir_branch, key_value, &result); if (!result.sr_item.i_node_ref_size) { /* A leaf... */ if (result.sr_found) { - if (!idx_remove_branch_item_right(ot, ind, current, &iref, &result.sr_item)) - goto failed; + if (ind->mi_lazy_delete) { + /* If the we have a W lock, then fetch decided that we + * need to compact the page. + * The decision is made by xt_idx_lazy_delete_on_leaf() + */ + if (!iref.ir_xlock) + idx_lazy_delete_branch_item(ot, ind, &iref, &result.sr_item); + else { + if (!iref.ir_block->cp_del_count) { + if (!idx_remove_branch_item_right(ot, ind, current, &iref, &result.sr_item)) + goto failed; + } + else { + if (!idx_lazy_remove_leaf_item_right(ot, ind, &iref, &result.sr_item)) + goto failed; + } + } + } + else { + if (!idx_remove_branch_item_right(ot, ind, current, &iref, &result.sr_item)) + goto failed; + } } else - xt_ind_release(ot, ind, XT_UNLOCK_WRITE, &iref); + xt_ind_release(ot, ind, iref.ir_xlock ? XT_UNLOCK_WRITE : XT_UNLOCK_READ, &iref); goto done_ok; } if (!idx_push(&stack, current, &result.sr_item)) { @@ -1803,6 +2342,35 @@ static xtBool idx_delete(XTOpenTablePtr /* Must be a non-leaf!: */ ASSERT_NS(result.sr_item.i_node_ref_size); + if (ind->mi_lazy_delete) { + if (!idx_lazy_delete_on_node(ind, iref.ir_block, &result.sr_item)) { + /* We need to remove some items from this node: */ + + if (!lock_structure) { + xt_ind_release(ot, ind, XT_UNLOCK_READ, &iref); + XT_INDEX_UNLOCK(ind, ot); + lock_structure = TRUE; + goto lock_and_retry; + } + + idx_set_item_deleted(&iref, &result.sr_item); + if (!idx_remove_lazy_deleted_item_in_node(ot, ind, current, &iref, key_value)) + goto failed; + goto done_ok; + } + + if (!ot->ot_table->tab_dic.dic_no_lazy_delete) { + /* {LAZY-DEL-INDEX-ITEMS} + * We just set item to deleted, this is a significant time + * saver. + * But this item can only be cleaned up when all + * items on the node below are deleted. + */ + idx_lazy_delete_branch_item(ot, ind, &iref, &result.sr_item); + goto done_ok; + } + } + /* We will have to remove the key from a non-leaf node, * which means we are changing the structure of the index. * Make sure we have a structural lock: @@ -1815,86 +2383,8 @@ static xtBool idx_delete(XTOpenTablePtr } /* This is the item we will have to replace: */ - delete_node = idx_top(&stack); - - /* Follow the branch after this item: */ - idx_next_branch_item(ot->ot_table, ind, iref.ir_branch, &result); - ASSERT_NS(XT_NODE_ID(current)); - xt_ind_release(ot, ind, XT_UNLOCK_READ, &iref); - - /* Go down the left-hand side until we reach a leaf: */ - while (XT_NODE_ID(current)) { - current = result.sr_branch; - if (!xt_ind_fetch(ot, current, XT_XLOCK_LEAF, &iref)) - goto failed; - idx_first_branch_item(ot->ot_table, ind, iref.ir_branch, &result); - if (!result.sr_item.i_node_ref_size) - break; - xt_ind_release(ot, ind, XT_UNLOCK_READ, &iref); - if (!idx_push(&stack, current, &result.sr_item)) - goto failed; - } - - ASSERT_NS(XT_NODE_ID(current)); - ASSERT_NS(!result.sr_item.i_node_ref_size); - - if (!xt_ind_reserve(ot, stack.s_top + 2, iref.ir_branch)) { - xt_ind_release(ot, ind, XT_UNLOCK_WRITE, &iref); + if (!idx_remove_item_in_node(ot, ind, &stack, &iref, key_value)) goto failed; - } - - /* Crawl back up the stack trace, looking for a key - * that can be used to replace the deleted key. - * - * Any empty nodes on the way up can be removed! - */ - if (result.sr_item.i_total_size > 0) { - /* There is a key in the leaf, extract it, and put it in the node: */ - memcpy(key_value->sv_key, &iref.ir_branch->tb_data[result.sr_item.i_item_offset], result.sr_item.i_item_size); - /* This call also frees the iref.ir_branch page! */ - if (!idx_remove_branch_item_right(ot, ind, current, &iref, &result.sr_item)) - goto failed; - if (!idx_replace_node_key(ot, ind, delete_node, &stack, result.sr_item.i_item_size, key_value->sv_key)) - goto failed; - goto done_ok_2; - } - - xt_ind_release(ot, ind, XT_UNLOCK_WRITE, &iref); - - for (;;) { - /* The current node/leaf is empty, remove it: */ - idx_free_branch(ot, ind, current); - - current_top = idx_pop(&stack); - current = current_top->i_branch; - if (!xt_ind_fetch(ot, current, XT_XLOCK_LEAF, &iref)) - goto failed; - - if (current_top == delete_node) { - /* All children have been removed. Delete the key and done: */ - if (!idx_remove_branch_item_right(ot, ind, current, &iref, &current_top->i_pos)) - goto failed; - goto done_ok_2; - } - - if (current_top->i_pos.i_total_size > current_top->i_pos.i_node_ref_size) { - /* Save the key: */ - memcpy(key_value->sv_key, &iref.ir_branch->tb_data[current_top->i_pos.i_item_offset], current_top->i_pos.i_item_size); - /* This function also frees the cache page: */ - if (!idx_remove_branch_item_left(ot, ind, current, &iref, &current_top->i_pos)) - goto failed; - if (!idx_replace_node_key(ot, ind, delete_node, &stack, current_top->i_pos.i_item_size, key_value->sv_key)) - goto failed; - goto done_ok_2; - } - xt_ind_release(ot, ind, current_top->i_pos.i_node_ref_size ? XT_UNLOCK_READ : XT_UNLOCK_WRITE, &iref); - } - - - done_ok_2: -#ifdef XT_TRACK_INDEX_UPDATES - ASSERT_NS(ot->ot_ind_reserved >= ot->ot_ind_reads); -#endif done_ok: XT_INDEX_UNLOCK(ind, ot); @@ -1945,7 +2435,8 @@ xtPublic xtBool xt_idx_update_row_id(XTO xtWord1 key_buf[XT_INDEX_MAX_KEY_SIZE + XT_MAX_RECORD_REF_SIZE]; #ifdef DEBUG - iref.ir_ulock = XT_UNLOCK_NONE; + iref.ir_xlock = 2; + iref.ir_updated = 2; #endif #ifdef CHECK_AND_PRINT idx_check_index(ot, ind, TRUE); @@ -1989,7 +2480,7 @@ xtPublic xtBool xt_idx_update_row_id(XTO goto done_ok; while (XT_NODE_ID(current)) { - if (!xt_ind_fetch(ot, current, XT_LOCK_READ, &iref)) + if (!xt_ind_fetch(ot, ind, current, XT_LOCK_READ, &iref)) goto failed; ind->mi_scan_branch(ot->ot_table, ind, iref.ir_branch, &key_value, &result); if (result.sr_found || !result.sr_item.i_node_ref_size) @@ -1999,23 +2490,10 @@ xtPublic xtBool xt_idx_update_row_id(XTO } if (result.sr_found) { - size_t offset; - xtWord1 *data; - - offset = - /* This is the offset of the reference in the item we found: */ - result.sr_item.i_item_offset + result.sr_item.i_item_size - XT_RECORD_REF_SIZE + - /* This is the offset of the row id in the reference: */ - 4; - data = &iref.ir_branch->tb_data[offset]; - - /* This update does not change the structure of page, so we do it without - * copying the page before we write. - * - * TODO: Check that concurrent reads can handle this! + /* TODO: Check that concurrent reads can handle this! * assuming the write is not atomic. */ - XT_SET_DISK_4(data, row_id); + idx_set_item_row_id(&iref, &result.sr_item, row_id); xt_ind_release(ot, ind, XT_UNLOCK_R_UPDATE, &iref); } else @@ -2076,7 +2554,8 @@ xtPublic xtBool xt_idx_search(XTOpenTabl XTIdxResultRec result; #ifdef DEBUG - iref.ir_ulock = XT_UNLOCK_NONE; + iref.ir_xlock = 2; + iref.ir_updated = 2; #endif if (ot->ot_ind_rhandle) { xt_ind_release_handle(ot->ot_ind_rhandle, FALSE, ot->ot_thread); @@ -2110,7 +2589,7 @@ xtPublic xtBool xt_idx_search(XTOpenTabl goto done_ok; while (XT_NODE_ID(current)) { - if (!xt_ind_fetch(ot, current, XT_LOCK_READ, &iref)) + if (!xt_ind_fetch(ot, ind, current, XT_LOCK_READ, &iref)) goto failed; ind->mi_scan_branch(ot->ot_table, ind, iref.ir_branch, &search_key->sk_key_value, &result); if (result.sr_found) @@ -2124,6 +2603,17 @@ xtPublic xtBool xt_idx_search(XTOpenTabl current = result.sr_branch; } + if (ind->mi_lazy_delete) { + ignore_lazy_deleted_items: + while (result.sr_item.i_item_offset < result.sr_item.i_total_size) { + if (result.sr_row_id != (xtRowID) -1) { + idx_still_on_key(ind, search_key, iref.ir_branch, &result.sr_item); + break; + } + idx_next_branch_item(ot->ot_table, ind, iref.ir_branch, &result); + } + } + if (result.sr_item.i_item_offset == result.sr_item.i_total_size) { IdxStackItemPtr node; @@ -2134,12 +2624,39 @@ xtPublic xtBool xt_idx_search(XTOpenTabl xt_ind_release(ot, ind, XT_UNLOCK_READ, &iref); while ((node = idx_pop(&stack))) { if (node->i_pos.i_item_offset < node->i_pos.i_total_size) { - xtRecordID rec_id; - - if (!xt_ind_fetch(ot, node->i_branch, XT_LOCK_READ, &iref)) + if (!xt_ind_fetch(ot, ind, node->i_branch, XT_LOCK_READ, &iref)) goto failed; - xt_get_record_ref(&iref.ir_branch->tb_data[node->i_pos.i_item_offset + node->i_pos.i_item_size - XT_RECORD_REF_SIZE], &rec_id, &ot->ot_curr_row_id); - ot->ot_curr_rec_id = rec_id; + xt_get_res_record_ref(&iref.ir_branch->tb_data[node->i_pos.i_item_offset + node->i_pos.i_item_size - XT_RECORD_REF_SIZE], &result); + + if (ind->mi_lazy_delete) { + result.sr_item = node->i_pos; + if (result.sr_row_id == (xtRowID) -1) { + /* If this node pointer is lazy deleted, then + * go down the next branch... + */ + idx_next_branch_item(ot->ot_table, ind, iref.ir_branch, &result); + + /* Go down to the bottom: */ + current = node->i_branch; + while (XT_NODE_ID(current)) { + xt_ind_release(ot, ind, XT_UNLOCK_READ, &iref); + if (!idx_push(&stack, current, &result.sr_item)) + goto failed; + current = result.sr_branch; + if (!xt_ind_fetch(ot, ind, current, XT_LOCK_READ, &iref)) + goto failed; + idx_first_branch_item(ot->ot_table, ind, iref.ir_branch, &result); + if (!result.sr_item.i_node_ref_size) + break; + } + + goto ignore_lazy_deleted_items; + } + idx_still_on_key(ind, search_key, iref.ir_branch, &result.sr_item); + } + + ot->ot_curr_rec_id = result.sr_rec_id; + ot->ot_curr_row_id = result.sr_row_id; ot->ot_ind_state = node->i_pos; /* Convert the pointer to a handle which can be used in later operations: */ @@ -2180,14 +2697,16 @@ xtPublic xtBool xt_idx_search(XTOpenTabl //idx_check_index(ot, ind, TRUE); //idx_check_on_key(ot); #endif - ASSERT_NS(iref.ir_ulock == XT_UNLOCK_NONE); + ASSERT_NS(iref.ir_xlock == 2); + ASSERT_NS(iref.ir_updated == 2); return OK; failed: XT_INDEX_UNLOCK(ind, ot); if (idx_out_of_memory_failure(ot)) goto retry_after_oom; - ASSERT_NS(iref.ir_ulock == XT_UNLOCK_NONE); + ASSERT_NS(iref.ir_xlock == 2); + ASSERT_NS(iref.ir_updated == 2); return FAILED; } @@ -2199,7 +2718,8 @@ xtPublic xtBool xt_idx_search_prev(XTOpe XTIdxResultRec result; #ifdef DEBUG - iref.ir_ulock = XT_UNLOCK_NONE; + iref.ir_xlock = 2; + iref.ir_updated = 2; #endif if (ot->ot_ind_rhandle) { xt_ind_release_handle(ot->ot_ind_rhandle, FALSE, ot->ot_thread); @@ -2232,7 +2752,7 @@ xtPublic xtBool xt_idx_search_prev(XTOpe goto done_ok; while (XT_NODE_ID(current)) { - if (!xt_ind_fetch(ot, current, XT_LOCK_READ, &iref)) + if (!xt_ind_fetch(ot, ind, current, XT_LOCK_READ, &iref)) goto failed; ind->mi_scan_branch(ot->ot_table, ind, iref.ir_branch, &search_key->sk_key_value, &result); if (result.sr_found) @@ -2249,17 +2769,43 @@ xtPublic xtBool xt_idx_search_prev(XTOpe if (result.sr_item.i_item_offset == 0) { IdxStackItemPtr node; - /* We are at the end of a leaf node. - * Go up the stack to find the start poition of the next key. + search_up_stack: + /* We are at the start of a leaf node. + * Go up the stack to find the start position of the next key. * If we find none, then we are the end of the index. */ xt_ind_release(ot, ind, XT_UNLOCK_READ, &iref); while ((node = idx_pop(&stack))) { if (node->i_pos.i_item_offset > node->i_pos.i_node_ref_size) { - if (!xt_ind_fetch(ot, node->i_branch, XT_LOCK_READ, &iref)) + if (!xt_ind_fetch(ot, ind, node->i_branch, XT_LOCK_READ, &iref)) goto failed; result.sr_item = node->i_pos; ind->mi_prev_item(ot->ot_table, ind, iref.ir_branch, &result); + + if (ind->mi_lazy_delete) { + if (result.sr_row_id == (xtRowID) -1) { + /* Go down to the bottom, in order to scan the leaf backwards: */ + current = node->i_branch; + while (XT_NODE_ID(current)) { + xt_ind_release(ot, ind, XT_UNLOCK_READ, &iref); + if (!idx_push(&stack, current, &result.sr_item)) + goto failed; + current = result.sr_branch; + if (!xt_ind_fetch(ot, ind, current, XT_LOCK_READ, &iref)) + goto failed; + ind->mi_last_item(ot->ot_table, ind, iref.ir_branch, &result); + if (!result.sr_item.i_node_ref_size) + break; + } + + /* If the leaf empty we have to go up the stack again... */ + if (result.sr_item.i_total_size == 0) + goto search_up_stack; + + goto scan_back_in_leaf; + } + } + goto record_found; } } @@ -2269,6 +2815,16 @@ xtPublic xtBool xt_idx_search_prev(XTOpe /* We must just step once to the left in this leaf node... */ ind->mi_prev_item(ot->ot_table, ind, iref.ir_branch, &result); + if (ind->mi_lazy_delete) { + scan_back_in_leaf: + while (result.sr_row_id == (xtRowID) -1) { + if (result.sr_item.i_item_offset == 0) + goto search_up_stack; + ind->mi_prev_item(ot->ot_table, ind, iref.ir_branch, &result); + } + idx_still_on_key(ind, search_key, iref.ir_branch, &result.sr_item); + } + record_found: ot->ot_curr_rec_id = result.sr_rec_id; ot->ot_curr_row_id = result.sr_row_id; @@ -2330,34 +2886,47 @@ xtPublic xtBool xt_idx_next(register XTO XTIndReferenceRec iref; #ifdef DEBUG - iref.ir_ulock = XT_UNLOCK_NONE; + iref.ir_xlock = 2; + iref.ir_updated = 2; #endif ASSERT_NS(ot->ot_ind_rhandle); xt_ind_lock_handle(ot->ot_ind_rhandle); - if (!ot->ot_ind_state.i_node_ref_size && - ot->ot_ind_state.i_item_offset < ot->ot_ind_state.i_total_size && + result.sr_item = ot->ot_ind_state; + if (!result.sr_item.i_node_ref_size && + result.sr_item.i_item_offset < result.sr_item.i_total_size && ot->ot_ind_rhandle->ih_cache_reference) { - key_value.sv_key = &ot->ot_ind_rhandle->ih_branch->tb_data[ot->ot_ind_state.i_item_offset]; - key_value.sv_length = ot->ot_ind_state.i_item_size - XT_RECORD_REF_SIZE; + XTIdxItemRec prev_item; + + key_value.sv_key = &ot->ot_ind_rhandle->ih_branch->tb_data[result.sr_item.i_item_offset]; + key_value.sv_length = result.sr_item.i_item_size - XT_RECORD_REF_SIZE; - result.sr_item = ot->ot_ind_state; + prev_item = result.sr_item; idx_next_branch_item(ot->ot_table, ind, ot->ot_ind_rhandle->ih_branch, &result); + + if (ind->mi_lazy_delete) { + while (result.sr_item.i_item_offset < result.sr_item.i_total_size) { + if (result.sr_row_id != (xtRowID) -1) + break; + prev_item = result.sr_item; + idx_next_branch_item(ot->ot_table, ind, ot->ot_ind_rhandle->ih_branch, &result); + } + } + if (result.sr_item.i_item_offset < result.sr_item.i_total_size) { /* Still on key? */ - if (search_key && search_key->sk_on_key) { - search_key->sk_on_key = myxt_compare_key(ind, search_key->sk_key_value.sv_flags, search_key->sk_key_value.sv_length, - search_key->sk_key_value.sv_key, &ot->ot_ind_rhandle->ih_branch->tb_data[result.sr_item.i_item_offset]) == 0; - } + idx_still_on_key(ind, search_key, ot->ot_ind_rhandle->ih_branch, &result.sr_item); xt_ind_unlock_handle(ot->ot_ind_rhandle); goto checked_on_key; } + + result.sr_item = prev_item; } key_value.sv_flags = XT_SEARCH_WHOLE_KEY; - xt_get_record_ref(&ot->ot_ind_rhandle->ih_branch->tb_data[ot->ot_ind_state.i_item_offset + ot->ot_ind_state.i_item_size - XT_RECORD_REF_SIZE], &key_value.sv_rec_id, &key_value.sv_row_id); + xt_get_record_ref(&ot->ot_ind_rhandle->ih_branch->tb_data[result.sr_item.i_item_offset + result.sr_item.i_item_size - XT_RECORD_REF_SIZE], &key_value.sv_rec_id, &key_value.sv_row_id); key_value.sv_key = key_buf; - key_value.sv_length = ot->ot_ind_state.i_item_size - XT_RECORD_REF_SIZE; - memcpy(key_buf, &ot->ot_ind_rhandle->ih_branch->tb_data[ot->ot_ind_state.i_item_offset], key_value.sv_length); + key_value.sv_length = result.sr_item.i_item_size - XT_RECORD_REF_SIZE; + memcpy(key_buf, &ot->ot_ind_rhandle->ih_branch->tb_data[result.sr_item.i_item_offset], key_value.sv_length); xt_ind_release_handle(ot->ot_ind_rhandle, TRUE, ot->ot_thread); ot->ot_ind_rhandle = NULL; @@ -2375,7 +2944,7 @@ xtPublic xtBool xt_idx_next(register XTO } while (XT_NODE_ID(current)) { - if (!xt_ind_fetch(ot, current, XT_LOCK_READ, &iref)) + if (!xt_ind_fetch(ot, ind, current, XT_LOCK_READ, &iref)) goto failed; ind->mi_scan_branch(ot->ot_table, ind, iref.ir_branch, &key_value, &result); if (result.sr_item.i_node_ref_size) { @@ -2389,7 +2958,7 @@ xtPublic xtBool xt_idx_next(register XTO if (!idx_push(&stack, current, &result.sr_item)) goto failed; current = result.sr_branch; - if (!xt_ind_fetch(ot, current, XT_LOCK_READ, &iref)) + if (!xt_ind_fetch(ot, ind, current, XT_LOCK_READ, &iref)) goto failed; idx_first_branch_item(ot->ot_table, ind, iref.ir_branch, &result); if (!result.sr_item.i_node_ref_size) @@ -2416,6 +2985,15 @@ xtPublic xtBool xt_idx_next(register XTO current = result.sr_branch; } + if (ind->mi_lazy_delete) { + ignore_lazy_deleted_items: + while (result.sr_item.i_item_offset < result.sr_item.i_total_size) { + if (result.sr_row_id != (xtRowID) -1) + break; + idx_next_branch_item(NULL, ind, iref.ir_branch, &result); + } + } + /* Check the current position in a leaf: */ if (result.sr_item.i_item_offset == result.sr_item.i_total_size) { /* At the end: */ @@ -2428,10 +3006,37 @@ xtPublic xtBool xt_idx_next(register XTO xt_ind_release(ot, ind, XT_UNLOCK_READ, &iref); while ((node = idx_pop(&stack))) { if (node->i_pos.i_item_offset < node->i_pos.i_total_size) { - if (!xt_ind_fetch(ot, node->i_branch, XT_LOCK_READ, &iref)) + if (!xt_ind_fetch(ot, ind, node->i_branch, XT_LOCK_READ, &iref)) goto failed; result.sr_item = node->i_pos; xt_get_res_record_ref(&iref.ir_branch->tb_data[result.sr_item.i_item_offset + result.sr_item.i_item_size - XT_RECORD_REF_SIZE], &result); + + if (ind->mi_lazy_delete) { + if (result.sr_row_id == (xtRowID) -1) { + /* If this node pointer is lazy deleted, then + * go down the next branch... + */ + idx_next_branch_item(ot->ot_table, ind, iref.ir_branch, &result); + + /* Go down to the bottom: */ + current = node->i_branch; + while (XT_NODE_ID(current)) { + xt_ind_release(ot, ind, XT_UNLOCK_READ, &iref); + if (!idx_push(&stack, current, &result.sr_item)) + goto failed; + current = result.sr_branch; + if (!xt_ind_fetch(ot, ind, current, XT_LOCK_READ, &iref)) + goto failed; + idx_first_branch_item(ot->ot_table, ind, iref.ir_branch, &result); + if (!result.sr_item.i_node_ref_size) + break; + } + + /* And scan the leaf... */ + goto ignore_lazy_deleted_items; + } + } + goto unlock_check_on_key; } } @@ -2503,32 +3108,39 @@ xtPublic xtBool xt_idx_prev(register XTO IdxStackItemPtr node; #ifdef DEBUG - iref.ir_ulock = XT_UNLOCK_NONE; + iref.ir_xlock = 2; + iref.ir_updated = 2; #endif ASSERT_NS(ot->ot_ind_rhandle); xt_ind_lock_handle(ot->ot_ind_rhandle); - if (!ot->ot_ind_state.i_node_ref_size && ot->ot_ind_state.i_item_offset > 0) { - key_value.sv_key = &ot->ot_ind_rhandle->ih_branch->tb_data[ot->ot_ind_state.i_item_offset]; - key_value.sv_length = ot->ot_ind_state.i_item_size - XT_RECORD_REF_SIZE; + result.sr_item = ot->ot_ind_state; + if (!result.sr_item.i_node_ref_size && result.sr_item.i_item_offset > 0) { + key_value.sv_key = &ot->ot_ind_rhandle->ih_branch->tb_data[result.sr_item.i_item_offset]; + key_value.sv_length = result.sr_item.i_item_size - XT_RECORD_REF_SIZE; - result.sr_item = ot->ot_ind_state; ind->mi_prev_item(ot->ot_table, ind, ot->ot_ind_rhandle->ih_branch, &result); - if (search_key && search_key->sk_on_key) { - search_key->sk_on_key = myxt_compare_key(ind, search_key->sk_key_value.sv_flags, search_key->sk_key_value.sv_length, - search_key->sk_key_value.sv_key, &ot->ot_ind_rhandle->ih_branch->tb_data[result.sr_item.i_item_offset]) == 0; + if (ind->mi_lazy_delete) { + while (result.sr_row_id == (xtRowID) -1) { + if (result.sr_item.i_item_offset == 0) + goto research; + ind->mi_prev_item(ot->ot_table, ind, ot->ot_ind_rhandle->ih_branch, &result); + } } + idx_still_on_key(ind, search_key, ot->ot_ind_rhandle->ih_branch, &result.sr_item); + xt_ind_unlock_handle(ot->ot_ind_rhandle); goto checked_on_key; } + research: key_value.sv_flags = XT_SEARCH_WHOLE_KEY; key_value.sv_rec_id = ot->ot_curr_rec_id; key_value.sv_row_id = 0; key_value.sv_key = key_buf; - key_value.sv_length = ot->ot_ind_state.i_item_size - XT_RECORD_REF_SIZE; - memcpy(key_buf, &ot->ot_ind_rhandle->ih_branch->tb_data[ot->ot_ind_state.i_item_offset], key_value.sv_length); + key_value.sv_length = result.sr_item.i_item_size - XT_RECORD_REF_SIZE; + memcpy(key_buf, &ot->ot_ind_rhandle->ih_branch->tb_data[result.sr_item.i_item_offset], key_value.sv_length); xt_ind_release_handle(ot->ot_ind_rhandle, TRUE, ot->ot_thread); ot->ot_ind_rhandle = NULL; @@ -2546,29 +3158,39 @@ xtPublic xtBool xt_idx_prev(register XTO } while (XT_NODE_ID(current)) { - if (!xt_ind_fetch(ot, current, XT_LOCK_READ, &iref)) + if (!xt_ind_fetch(ot, ind, current, XT_LOCK_READ, &iref)) goto failed; ind->mi_scan_branch(ot->ot_table, ind, iref.ir_branch, &key_value, &result); if (result.sr_item.i_node_ref_size) { if (result.sr_found) { /* If we have found the key in a node: */ + search_down_stack: /* Go down to the bottom: */ while (XT_NODE_ID(current)) { xt_ind_release(ot, ind, XT_UNLOCK_READ, &iref); if (!idx_push(&stack, current, &result.sr_item)) goto failed; current = result.sr_branch; - if (!xt_ind_fetch(ot, current, XT_LOCK_READ, &iref)) + if (!xt_ind_fetch(ot, ind, current, XT_LOCK_READ, &iref)) goto failed; ind->mi_last_item(ot->ot_table, ind, iref.ir_branch, &result); if (!result.sr_item.i_node_ref_size) break; } - /* Is the leaf not empty, then we are done... */ + /* If the leaf empty we have to go up the stack again... */ if (result.sr_item.i_total_size == 0) break; + + if (ind->mi_lazy_delete) { + while (result.sr_row_id == (xtRowID) -1) { + if (result.sr_item.i_item_offset == 0) + goto search_up_stack; + ind->mi_prev_item(ot->ot_table, ind, iref.ir_branch, &result); + } + } + goto unlock_check_on_key; } } @@ -2580,6 +3202,15 @@ xtPublic xtBool xt_idx_prev(register XTO if (result.sr_item.i_item_offset == 0) break; ind->mi_prev_item(ot->ot_table, ind, iref.ir_branch, &result); + + if (ind->mi_lazy_delete) { + while (result.sr_row_id == (xtRowID) -1) { + if (result.sr_item.i_item_offset == 0) + goto search_up_stack; + ind->mi_prev_item(ot->ot_table, ind, iref.ir_branch, &result); + } + } + goto unlock_check_on_key; } xt_ind_release(ot, ind, XT_UNLOCK_READ, &iref); @@ -2588,6 +3219,7 @@ xtPublic xtBool xt_idx_prev(register XTO current = result.sr_branch; } + search_up_stack: /* We are at the start of a leaf node. * Go up the stack to find the start poition of the next key. * If we find none, then we are the end of the index. @@ -2595,10 +3227,18 @@ xtPublic xtBool xt_idx_prev(register XTO xt_ind_release(ot, ind, XT_UNLOCK_READ, &iref); while ((node = idx_pop(&stack))) { if (node->i_pos.i_item_offset > node->i_pos.i_node_ref_size) { - if (!xt_ind_fetch(ot, node->i_branch, XT_LOCK_READ, &iref)) + if (!xt_ind_fetch(ot, ind, node->i_branch, XT_LOCK_READ, &iref)) goto failed; result.sr_item = node->i_pos; ind->mi_prev_item(ot->ot_table, ind, iref.ir_branch, &result); + + if (ind->mi_lazy_delete) { + if (result.sr_row_id == (xtRowID) -1) { + current = node->i_branch; + goto search_down_stack; + } + } + goto unlock_check_on_key; } } @@ -2648,7 +3288,7 @@ xtPublic xtBool xt_idx_prev(register XTO } /* Return TRUE if the record matches the current index search! */ -xtPublic xtBool xt_idx_match_search(register XTOpenTablePtr ot __attribute__((unused)), register XTIndexPtr ind, register XTIdxSearchKeyPtr search_key, xtWord1 *buf, int mode) +xtPublic xtBool xt_idx_match_search(register XTOpenTablePtr XT_UNUSED(ot), register XTIndexPtr ind, register XTIdxSearchKeyPtr search_key, xtWord1 *buf, int mode) { int r; xtWord1 key_buf[XT_INDEX_MAX_KEY_SIZE]; @@ -2666,7 +3306,7 @@ xtPublic xtBool xt_idx_match_search(regi return FALSE; } -static void idx_set_index_selectivity(XTThreadPtr self __attribute__((unused)), XTOpenTablePtr ot, XTIndexPtr ind) +static void idx_set_index_selectivity(XTThreadPtr self, XTOpenTablePtr ot, XTIndexPtr ind) { static const xtRecordID MAX_RECORDS = 100; @@ -2784,7 +3424,7 @@ static void idx_set_index_selectivity(XT ot->ot_ind_rhandle = NULL; failed: - ot->ot_table->tab_dic.dic_disable_index = XT_INDEX_CORRUPTED; + xt_tab_disable_index(ot->ot_table, XT_INDEX_CORRUPTED); xt_log_and_clear_exception_ns(); return; } @@ -2834,10 +3474,11 @@ static u_int idx_check_node(XTOpenTableP XTIndReferenceRec iref; #ifdef DEBUG - iref.ir_ulock = XT_UNLOCK_NONE; + iref.ir_xlock = 2; + iref.ir_updated = 2; #endif ASSERT_NS(XT_NODE_ID(node) <= XT_NODE_ID(ot->ot_table->tab_ind_eof)); - if (!xt_ind_fetch(ot, node, XT_LOCK_READ, &iref)) + if (!xt_ind_fetch(ot, ind, node, XT_LOCK_READ, &iref)) return 0; idx_first_branch_item(ot->ot_table, ind, iref.ir_branch, &result); @@ -2974,7 +3615,7 @@ xtPublic void xt_check_indices(XTOpenTab track_block_exists(current); #endif printf("%d ", (int) XT_NODE_ID(current)); - if (!xt_ind_read_bytes(ot, current, sizeof(XTIndFreeBlockRec), (xtWord1 *) &free_block)) { + if (!xt_ind_read_bytes(ot, *ind, current, sizeof(XTIndFreeBlockRec), (xtWord1 *) &free_block)) { xt_log_and_clear_exception_ns(); break; } @@ -3000,6 +3641,88 @@ xtPublic void xt_check_indices(XTOpenTab /* * ----------------------------------------------------------------------- + * Load index + */ + +static void idx_load_node(XTThreadPtr self, XTOpenTablePtr ot, XTIndexPtr ind, xtIndexNodeID node) +{ + XTIdxResultRec result; + XTIndReferenceRec iref; + + ASSERT_NS(XT_NODE_ID(node) <= XT_NODE_ID(ot->ot_table->tab_ind_eof)); + if (!xt_ind_fetch(ot, ind, node, XT_LOCK_READ, &iref)) + xt_throw(self); + + idx_first_branch_item(ot->ot_table, ind, iref.ir_branch, &result); + if (result.sr_item.i_node_ref_size) + idx_load_node(self, ot, ind, result.sr_branch); + while (result.sr_item.i_item_offset < result.sr_item.i_total_size) { + idx_next_branch_item(ot->ot_table, ind, iref.ir_branch, &result); + if (result.sr_item.i_node_ref_size) + idx_load_node(self, ot, ind, result.sr_branch); + } + + xt_ind_release(ot, ind, XT_UNLOCK_READ, &iref); +} + +xtPublic void xt_load_indices(XTThreadPtr self, XTOpenTablePtr ot) +{ + register XTTableHPtr tab = ot->ot_table; + XTIndexPtr *ind_ptr; + XTIndexPtr ind; + xtIndexNodeID current; + + xt_lock_mutex(self, &tab->tab_ind_flush_lock); + pushr_(xt_unlock_mutex, &tab->tab_ind_flush_lock); + + ind_ptr = tab->tab_dic.dic_keys; + for (u_int k=0; k<tab->tab_dic.dic_key_count; k++, ind_ptr++) { + ind = *ind_ptr; + XT_INDEX_WRITE_LOCK(ind, ot); + if ((XT_NODE_ID(current) = XT_NODE_ID(ind->mi_root))) + idx_load_node(self, ot, ind, current); + XT_INDEX_UNLOCK(ind, ot); + } + + freer_(); // xt_unlock_mutex(&tab->tab_ind_flush_lock) +} + +/* + * ----------------------------------------------------------------------- + * Count the number of deleted entries in a node: + */ + +/* + * {LAZY-DEL-INDEX-ITEMS} + * + * Use this function to count the number of deleted items + * in a node when it is loaded. + * + * The count helps us decide of the node should be "packed". + */ +xtPublic void xt_ind_count_deleted_items(XTTableHPtr tab, XTIndexPtr ind, XTIndBlockPtr block) +{ + XTIdxResultRec result; + int del_count = 0; + xtWord2 branch_size; + + branch_size = XT_GET_DISK_2(((XTIdxBranchDPtr) block->cb_data)->tb_size_2); + + /* This is possible when reading free pages. */ + if (XT_GET_INDEX_BLOCK_LEN(branch_size) < 2 || XT_GET_INDEX_BLOCK_LEN(branch_size) > XT_INDEX_PAGE_SIZE) + return; + + idx_first_branch_item(tab, ind, (XTIdxBranchDPtr) block->cb_data, &result); + while (result.sr_item.i_item_offset < result.sr_item.i_total_size) { + if (result.sr_row_id == (xtRowID) -1) + del_count++; + idx_next_branch_item(tab, ind, (XTIdxBranchDPtr) block->cb_data, &result); + } + block->cp_del_count = del_count; +} + +/* + * ----------------------------------------------------------------------- * Index consistant flush */ @@ -3408,7 +4131,7 @@ void XTIndexLogPool::ilp_init(struct XTT xt_throw(self); } -void XTIndexLogPool::ilp_close(struct XTThread *self __attribute__((unused)), xtBool lock) +void XTIndexLogPool::ilp_close(struct XTThread *XT_UNUSED(self), xtBool lock) { XTIndexLogPtr il; @@ -3570,7 +4293,7 @@ xtBool XTIndexLog::il_require_space(size return OK; } -xtBool XTIndexLog::il_write_byte(struct XTOpenTable *ot __attribute__((unused)), xtWord1 byte) +xtBool XTIndexLog::il_write_byte(struct XTOpenTable *ot, xtWord1 byte) { if (!il_require_space(1, ot->ot_thread)) return FAILED; @@ -3579,7 +4302,7 @@ xtBool XTIndexLog::il_write_byte(struct return OK; } -xtBool XTIndexLog::il_write_word4(struct XTOpenTable *ot __attribute__((unused)), xtWord4 value) +xtBool XTIndexLog::il_write_word4(struct XTOpenTable *ot, xtWord4 value) { xtWord1 *buffer; @@ -3591,7 +4314,7 @@ xtBool XTIndexLog::il_write_word4(struct return OK; } -xtBool XTIndexLog::il_write_block(struct XTOpenTable *ot __attribute__((unused)), XTIndBlockPtr block) +xtBool XTIndexLog::il_write_block(struct XTOpenTable *ot, XTIndBlockPtr block) { XTIndPageDataDPtr page_data; xtIndexNodeID node_id; @@ -3618,7 +4341,7 @@ xtBool XTIndexLog::il_write_block(struct return OK; } -xtBool XTIndexLog::il_write_header(struct XTOpenTable *ot __attribute__((unused)), size_t head_size, xtWord1 *head_buf) +xtBool XTIndexLog::il_write_header(struct XTOpenTable *ot, size_t head_size, xtWord1 *head_buf) { XTIndHeadDataDPtr head_data; === modified file 'storage/pbxt/src/index_xt.h' --- a/storage/pbxt/src/index_xt.h 2009-04-02 10:03:14 +0000 +++ b/storage/pbxt/src/index_xt.h 2009-08-17 11:12:36 +0000 @@ -24,6 +24,7 @@ #define __xt_index_h__ #ifdef DRIZZLED +#include <drizzled/definitions.h> #include <mysys/my_bitmap.h> #else #include <mysql_version.h> @@ -34,7 +35,6 @@ #include "linklist_xt.h" #include "datalog_xt.h" #include "datadic_xt.h" -//#include "cache_xt.h" #ifndef MYSQL_VERSION_ID #error MYSQL_VERSION_ID must be defined! @@ -109,7 +109,7 @@ class Field; #define XT_MAX_RECORD_REF_SIZE 8 -#define XT_INDEX_PAGE_DATA_SIZE XT_INDEX_PAGE_SIZE - 2 /* NOTE: 2 == offsetof(XTIdxBranchDRec, tb_data) */ +#define XT_INDEX_PAGE_DATA_SIZE (XT_INDEX_PAGE_SIZE - 2) /* NOTE: 2 == offsetof(XTIdxBranchDRec, tb_data) */ #define XT_MAKE_LEAF_SIZE(x) ((x) + offsetof(XTIdxBranchDRec, tb_data)) @@ -218,7 +218,7 @@ typedef struct XTIndFreeList { * in 32 threads on smalltab: runTest(SMALL_INSERT_TEST, 32, dbUrl) */ /* - * XT_INDEX_USE_RW_MUTEX: + * XT_INDEX_USE_RWMUTEX: * But the RW mutex is a close second, if not just as fast. * If it is at least as fast, then it is better because read lock * overhead is then zero. @@ -240,17 +240,24 @@ typedef struct XTIndFreeList { * Latest test show that RW mutex is slightly faster: * 127460 to 123574 payment transactions. */ -#define XT_INDEX_USE_RW_MUTEX + +#ifdef XT_NO_ATOMICS +#define XT_INDEX_USE_PTHREAD_RW +#else +//#define XT_INDEX_USE_RWMUTEX //#define XT_INDEX_USE_PTHREAD_RW +//#define XT_INDEX_SPINXSLOCK +#define XT_TAB_ROW_USE_XSMUTEX +#endif -#ifdef XT_INDEX_USE_FASTWRLOCK -#define XT_INDEX_LOCK_TYPE XTFastRWLockRec -#define XT_INDEX_INIT_LOCK(s, i) xt_fastrwlock_init(s, &(i)->mi_rwlock) -#define XT_INDEX_FREE_LOCK(s, i) xt_fastrwlock_free(s, &(i)->mi_rwlock) -#define XT_INDEX_READ_LOCK(i, o) xt_fastrwlock_slock(&(i)->mi_rwlock, (o)->ot_thread) -#define XT_INDEX_WRITE_LOCK(i, o) xt_fastrwlock_xlock(&(i)->mi_rwlock, (o)->ot_thread) -#define XT_INDEX_UNLOCK(i, o) xt_fastrwlock_unlock(&(i)->mi_rwlock, (o)->ot_thread) -#define XT_INDEX_HAVE_XLOCK(i, o) TRUE +#ifdef XT_TAB_ROW_USE_XSMUTEX +#define XT_INDEX_LOCK_TYPE XTXSMutexRec +#define XT_INDEX_INIT_LOCK(s, i) xt_xsmutex_init_with_autoname(s, &(i)->mi_rwlock) +#define XT_INDEX_FREE_LOCK(s, i) xt_xsmutex_free(s, &(i)->mi_rwlock) +#define XT_INDEX_READ_LOCK(i, o) xt_xsmutex_slock(&(i)->mi_rwlock, (o)->ot_thread->t_id) +#define XT_INDEX_WRITE_LOCK(i, o) xt_xsmutex_xlock(&(i)->mi_rwlock, (o)->ot_thread->t_id) +#define XT_INDEX_UNLOCK(i, o) xt_xsmutex_unlock(&(i)->mi_rwlock, (o)->ot_thread->t_id) +#define XT_INDEX_HAVE_XLOCK(i, o) ((i)->sxs_xlocker == (o)->ot_thread->t_id) #elif defined(XT_INDEX_USE_PTHREAD_RW) #define XT_INDEX_LOCK_TYPE xt_rwlock_type #define XT_INDEX_INIT_LOCK(s, i) xt_init_rwlock_with_autoname(s, &(i)->mi_rwlock) @@ -259,7 +266,15 @@ typedef struct XTIndFreeList { #define XT_INDEX_WRITE_LOCK(i, o) xt_xlock_rwlock_ns(&(i)->mi_rwlock) #define XT_INDEX_UNLOCK(i, o) xt_unlock_rwlock_ns(&(i)->mi_rwlock) #define XT_INDEX_HAVE_XLOCK(i, o) TRUE -#else // XT_INDEX_USE_RW_MUTEX +#elif defined(XT_INDEX_SPINXSLOCK) +#define XT_INDEX_LOCK_TYPE XTSpinXSLockRec +#define XT_INDEX_INIT_LOCK(s, i) xt_spinxslock_init_with_autoname(s, &(i)->mi_rwlock) +#define XT_INDEX_FREE_LOCK(s, i) xt_spinxslock_free(s, &(i)->mi_rwlock) +#define XT_INDEX_READ_LOCK(i, o) xt_spinxslock_slock(&(i)->mi_rwlock, (o)->ot_thread->t_id) +#define XT_INDEX_WRITE_LOCK(i, o) xt_spinxslock_xlock(&(i)->mi_rwlock, (o)->ot_thread->t_id) +#define XT_INDEX_UNLOCK(i, o) xt_spinxslock_unlock(&(i)->mi_rwlock, (o)->ot_thread->t_id) +#define XT_INDEX_HAVE_XLOCK(i, o) ((i)->mi_rwlock.nrw_xlocker == (o)->ot_thread->t_id) +#else // XT_INDEX_USE_RWMUTEX #define XT_INDEX_LOCK_TYPE XTRWMutexRec #define XT_INDEX_INIT_LOCK(s, i) xt_rwmutex_init_with_autoname(s, &(i)->mi_rwlock) #define XT_INDEX_FREE_LOCK(s, i) xt_rwmutex_free(s, &(i)->mi_rwlock) @@ -289,22 +304,24 @@ typedef struct XTIndex { XTIndFreeListPtr mi_free_list; /* List of free pages for this index. */ /* Protected by the mi_dirty_lock: */ - XTSpinLockRec mi_dirty_lock; /* Spin lock protecting the dirty & free lists. */ + XTSpinLockRec mi_dirty_lock; /* Spin lock protecting the dirty & free lists. */ struct XTIndBlock *mi_dirty_list; /* List of dirty pages for this index. */ u_int mi_dirty_blocks; /* Count of the dirty blocks. */ /* Index contants: */ u_int mi_flags; u_int mi_key_size; + u_int mi_max_items; /* The maximum number of items that can fit in a leaf node. */ xtBool mi_low_byte_first; xtBool mi_fix_key; + xtBool mi_lazy_delete; /* TRUE if index entries are "lazy deleted". */ u_int mi_single_type; /* Used when the index contains a single field. */ u_int mi_select_total; XTScanBranchFunc mi_scan_branch; XTPrevItemFunc mi_prev_item; XTLastItemFunc mi_last_item; XTSimpleCompFunc mi_simple_comp_key; - MY_BITMAP mi_col_map; /* Bit-map of columns in the index. */ + MX_BITMAP mi_col_map; /* Bit-map of columns in the index. */ u_int mi_subset_of; /* Indicates if this index is a complete subset of someother index. */ u_int mi_seg_count; XTIndexSegRec mi_seg[200]; @@ -344,6 +361,7 @@ typedef struct XTDictionary { Field **dic_blob_cols; /* MySQL related information. NULL when no tables are open from MySQL side! */ + xtBool dic_no_lazy_delete; /* FALSE if lazy delete is OK. */ u_int dic_disable_index; /* Non-zero if the index cannot be used. */ u_int dic_index_ver; /* The version of the index. */ u_int dic_key_count; @@ -462,6 +480,8 @@ xtBool xt_idx_prev(register struct XTOpe xtBool xt_idx_read(struct XTOpenTable *ot, struct XTIndex *ind, xtWord1 *rec_buf); void xt_ind_set_index_selectivity(XTThreadPtr self, struct XTOpenTable *ot); void xt_check_indices(struct XTOpenTable *ot); +void xt_load_indices(XTThreadPtr self, struct XTOpenTable *ot); +void xt_ind_count_deleted_items(struct XTTable *ot, struct XTIndex *ind, struct XTIndBlock *block); xtBool xt_flush_indices(struct XTOpenTable *ot, off_t *bytes_flushed, xtBool have_table_lock); void xt_ind_track_dump_block(struct XTTable *tab, xtIndexNodeID address); @@ -482,6 +502,7 @@ void xt_prev_branch_item_var(struct XTTa void xt_last_branch_item_fix(struct XTTable *tab, XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxResultPtr result); void xt_last_branch_item_var(struct XTTable *tab, XTIndexPtr ind, XTIdxBranchDPtr branch, register XTIdxResultPtr result); +xtBool xt_idx_lazy_delete_on_leaf(XTIndexPtr ind, struct XTIndBlock *block, xtWord2 branch_size); //#define TRACK_ACTIVITY #ifdef TRACK_ACTIVITY === modified file 'storage/pbxt/src/lock_xt.cc' --- a/storage/pbxt/src/lock_xt.cc 2009-04-02 10:03:14 +0000 +++ b/storage/pbxt/src/lock_xt.cc 2009-08-17 11:12:36 +0000 @@ -25,6 +25,10 @@ #include "xt_config.h" +#ifdef DRIZZLED +#include <bitset> +#endif + #include <stdio.h> #include "lock_xt.h" @@ -40,6 +44,16 @@ #endif /* + * This function should never be called. It indicates a link + * error! + */ +xtPublic void xt_log_atomic_error_and_abort(c_char *func, c_char *file, u_int line) +{ + xt_logf(NULL, func, file, line, XT_LOG_ERROR, "%s", "Atomic operations not supported\n"); + abort(); +} + +/* * ----------------------------------------------------------------------- * ROW LOCKS, LIST BASED */ @@ -715,7 +729,7 @@ xtBool xt_init_row_locks(XTRowLocksPtr r return OK; } -void xt_exit_row_locks(XTRowLocksPtr rl __attribute__((unused))) +void xt_exit_row_locks(XTRowLocksPtr rl) { for (int i=0; i<XT_ROW_LOCK_GROUP_COUNT; i++) { xt_spinlock_free(NULL, &rl->rl_groups[i].lg_lock); @@ -982,7 +996,7 @@ xtBool old_xt_init_row_locks(XTRowLocksP return OK; } -void old_xt_exit_row_locks(XTRowLocksPtr rl __attribute__((unused))) +void old_xt_exit_row_locks(XTRowLocksPtr XT_UNUSED(rl)) { } @@ -1007,10 +1021,6 @@ xtPublic void xt_exit_row_lock_list(XTRo * SPECIAL EXCLUSIVE/SHARED (XS) LOCK */ -#define XT_GET1(x) *(x) -#define XT_SET4(x, y) xt_atomic_set4(x, y) -#define XT_GET4(x) xt_atomic_get4(x) - #ifdef XT_THREAD_LOCK_INFO xtPublic void xt_rwmutex_init(struct XTThread *self, XTRWMutexPtr xsl, const char *n) #else @@ -1023,7 +1033,7 @@ xtPublic void xt_rwmutex_init(XTThreadPt #endif xt_init_mutex_with_autoname(self, &xsl->xs_lock); xt_init_cond(self, &xsl->xs_cond); - XT_SET4(&xsl->xs_state, 0); + xt_atomic_set4(&xsl->xs_state, 0); xsl->xs_xlocker = 0; /* Must be aligned! */ ASSERT(xt_thr_maximum_threads == xt_align_size(xt_thr_maximum_threads, XT_XS_LOCK_ALIGN)); @@ -1068,7 +1078,7 @@ xtPublic xtBool xt_rwmutex_xlock(XTRWMut } /* I am the locker (set state before locker!): */ - XT_SET4(&xsl->xs_state, 0); + xt_atomic_set4(&xsl->xs_state, 0); xsl->xs_xlocker = thd_id; /* Wait for all the read lockers: */ @@ -1078,7 +1088,7 @@ xtPublic xtBool xt_rwmutex_xlock(XTRWMut * Just in case of this, we keep the wait time down! */ if (!xt_timed_wait_cond_ns(&xsl->xs_cond, &xsl->xs_lock, 10)) { - XT_SET4(&xsl->xs_state, 0); + xt_atomic_set4(&xsl->xs_state, 0); xsl->xs_xlocker = 0; xt_unlock_mutex_ns(&xsl->xs_lock); return FAILED; @@ -1087,11 +1097,11 @@ xtPublic xtBool xt_rwmutex_xlock(XTRWMut /* State can be incremented in parallel by a reader * thread! */ - XT_SET4(&xsl->xs_state, xsl->xs_state + 1); + xt_atomic_set4(&xsl->xs_state, xsl->xs_state + 1); } /* I have waited for all: */ - XT_SET4(&xsl->xs_state, xt_thr_maximum_threads); + xt_atomic_set4(&xsl->xs_state, xt_thr_maximum_threads); #ifdef XT_THREAD_LOCK_INFO xt_thread_lock_info_add_owner(&xsl->xs_lock_info); @@ -1107,7 +1117,7 @@ xtPublic xtBool xt_rwmutex_slock(XTRWMut #endif ASSERT_NS(xt_get_self()->t_id == thd_id); - xt_flushed_inc1(&xsl->x.xs_rlock[thd_id]); + xt_atomic_inc1(&xsl->x.xs_rlock[thd_id]); if (xsl->x.xs_rlock[thd_id] > 1) return OK; @@ -1158,7 +1168,7 @@ xtPublic xtBool xt_rwmutex_unlock(XTRWMu /* I have an X lock. */ ASSERT_NS(xsl->x.xs_rlock[thd_id] == XT_NO_LOCK); ASSERT_NS(xsl->xs_state == xt_thr_maximum_threads); - XT_SET4(&xsl->xs_state, 0); + xt_atomic_set4(&xsl->xs_state, 0); xsl->xs_xlocker = 0; xt_unlock_mutex_ns(&xsl->xs_lock); /* Wake up any other X or shared lockers: */ @@ -1201,7 +1211,7 @@ xtPublic xtBool xt_rwmutex_unlock(XTRWMu return FAILED; } } - xt_flushed_dec1(&xsl->x.xs_rlock[thd_id]); + xt_atomic_dec1(&xsl->x.xs_rlock[thd_id]); xt_unlock_mutex_ns(&xsl->xs_lock); } else @@ -1213,7 +1223,7 @@ xtPublic xtBool xt_rwmutex_unlock(XTRWMu * try to get the lock xs_lock, I could hand for the duration * of the X lock. */ - xt_flushed_dec1(&xsl->x.xs_rlock[thd_id]); + xt_atomic_dec1(&xsl->x.xs_rlock[thd_id]); } } #ifdef XT_THREAD_LOCK_INFO @@ -1228,13 +1238,14 @@ xtPublic xtBool xt_rwmutex_unlock(XTRWMu */ #ifdef XT_THREAD_LOCK_INFO -xtPublic void xt_spinlock_init(XTThreadPtr self __attribute__((unused)), XTSpinLockPtr spl, const char *n) +xtPublic void xt_spinlock_init(XTThreadPtr self, XTSpinLockPtr spl, const char *n) #else -xtPublic void xt_spinlock_init(XTThreadPtr self __attribute__((unused)), XTSpinLockPtr spl) +xtPublic void xt_spinlock_init(XTThreadPtr self, XTSpinLockPtr spl) #endif { + (void) self; spl->spl_lock = 0; -#ifdef XT_SPL_DEFAULT +#ifdef XT_NO_ATOMICS xt_init_mutex(self, &spl->spl_mutex); #endif #ifdef DEBUG @@ -1246,9 +1257,10 @@ xtPublic void xt_spinlock_init(XTThreadP #endif } -xtPublic void xt_spinlock_free(XTThreadPtr self __attribute__((unused)), XTSpinLockPtr spl __attribute__((unused))) +xtPublic void xt_spinlock_free(XTThreadPtr XT_UNUSED(self), XTSpinLockPtr spl) { -#ifdef XT_SPL_DEFAULT + (void) spl; +#ifdef XT_NO_ATOMICS xt_free_mutex(&spl->spl_mutex); #endif #ifdef XT_THREAD_LOCK_INFO @@ -1266,7 +1278,7 @@ xtPublic xtBool xt_spinlock_spin(XTSpinL if (!*lck) { /* Try to get the lock: */ if (!xt_spinlock_set(spl)) - return OK; + goto done_ok; } } @@ -1274,6 +1286,7 @@ xtPublic xtBool xt_spinlock_spin(XTSpinL xt_critical_wait(); } + done_ok: return OK; } @@ -1400,147 +1413,96 @@ xtPublic void xt_fastlock_wakeup(XTFastL /* * ----------------------------------------------------------------------- * READ/WRITE SPIN LOCK + * + * An extremely genius very fast read/write lock based on atomics! */ #ifdef XT_THREAD_LOCK_INFO -xtPublic void xt_spinrwlock_init(struct XTThread *self, XTSpinRWLockPtr srw, const char *name) +xtPublic void xt_spinxslock_init(struct XTThread *XT_UNUSED(self), XTSpinXSLockPtr sxs, const char *name) #else -xtPublic void xt_spinrwlock_init(struct XTThread *self, XTSpinRWLockPtr srw) +xtPublic void xt_spinxslock_init(struct XTThread *XT_UNUSED(self), XTSpinXSLockPtr sxs) #endif { - xt_spinlock_init_with_autoname(self, &srw->srw_lock); - xt_spinlock_init_with_autoname(self, &srw->srw_state_lock); - srw->srw_state = 0; - srw->srw_xlocker = 0; - /* Must be aligned! */ - ASSERT(xt_thr_maximum_threads == xt_align_size(xt_thr_maximum_threads, XT_XS_LOCK_ALIGN)); - srw->x.srw_rlock = (xtWord1 *) xt_calloc(self, xt_thr_maximum_threads); + sxs->sxs_xlocked = 0; + sxs->sxs_rlock_count = 0; + sxs->sxs_wait_count = 0; +#ifdef DEBUG + sxs->sxs_locker = 0; +#endif #ifdef XT_THREAD_LOCK_INFO - srw->srw_name = name; - xt_thread_lock_info_init(&srw->srw_lock_info, srw); + sxs->sxs_name = name; + xt_thread_lock_info_init(&sxs->sxs_lock_info, sxs); #endif } -xtPublic void xt_spinrwlock_free(struct XTThread *self, XTSpinRWLockPtr srw) +xtPublic void xt_spinxslock_free(struct XTThread *XT_UNUSED(self), XTSpinXSLockPtr sxs) { - if (srw->x.srw_rlock) - xt_free(self, (void *) srw->x.srw_rlock); - xt_spinlock_free(self, &srw->srw_lock); - xt_spinlock_free(self, &srw->srw_state_lock); #ifdef XT_THREAD_LOCK_INFO - xt_thread_lock_info_free(&srw->srw_lock_info); + xt_thread_lock_info_free(&sxs->sxs_lock_info); +#else + (void) sxs; #endif } -xtPublic xtBool xt_spinrwlock_xlock(XTSpinRWLockPtr srw, xtThreadID thd_id) +xtPublic xtBool xt_spinxslock_xlock(XTSpinXSLockPtr sxs, xtThreadID XT_NDEBUG_UNUSED(thd_id)) { - xt_spinlock_lock(&srw->srw_lock); - ASSERT_NS(srw->x.srw_rlock[thd_id] == XT_NO_LOCK); - - xt_spinlock_lock(&srw->srw_state_lock); - - /* Set the state before xlocker (dirty read!) */ - srw->srw_state = 0; - - /* I am the locker: */ - srw->srw_xlocker = thd_id; + register xtWord2 set; - /* Wait for all the read lockers: */ - while (srw->srw_state < xt_thr_current_max_threads) { - while (srw->x.srw_rlock[srw->srw_state]) { - xt_spinlock_unlock(&srw->srw_state_lock); - /* Wait for this reader, during this time, the reader - * himself, may increment the state. */ - xt_critical_wait(); - xt_spinlock_lock(&srw->srw_state_lock); - } - /* State can be incremented in parallel by a reader - * thread! - */ - srw->srw_state++; + /* Wait for exclusive locker: */ + for (;;) { + set = xt_atomic_tas2(&sxs->sxs_xlocked, 1); + if (!set) + break; + xt_yield(); } - /* I have waited for all: */ - srw->srw_state = xt_thr_maximum_threads; +#ifdef DEBUG + sxs->sxs_locker = thd_id; +#endif - xt_spinlock_unlock(&srw->srw_state_lock); + /* Wait for all the reader to wait! */ + while (sxs->sxs_wait_count < sxs->sxs_rlock_count) + xt_yield(); #ifdef XT_THREAD_LOCK_INFO - xt_thread_lock_info_add_owner(&srw->srw_lock_info); + xt_thread_lock_info_add_owner(&sxs->sxs_lock_info); #endif - return OK; } -xtPublic xtBool xt_spinrwlock_slock(XTSpinRWLockPtr srw, xtThreadID thd_id) +xtPublic xtBool xt_spinxslock_slock(XTSpinXSLockPtr sxs) { - ASSERT_NS(srw->x.srw_rlock[thd_id] == XT_NO_LOCK); - srw->x.srw_rlock[thd_id] = XT_WANT_LOCK; + xt_atomic_inc2(&sxs->sxs_rlock_count); + /* Check if there could be an X locker: */ - if (srw->srw_xlocker) { - /* There is an X locker. - * If srw_state < thd_id then the X locker will wait for me. - * So I should not wait! - */ - if (srw->srw_state >= thd_id) { - /* If srw->srw_state >= thd_id, then the locker may have, or - * has already checked me, and I will have to wait. - * - * Otherwise, srw_state <= thd_id, which means the - * X locker has not checked me, and will still wait for me (or - * is already waiting for me). In this case, I will have to - * take the mutex to make sure exactly how far he - * is with the checking. - */ - xt_spinlock_lock(&srw->srw_state_lock); - while (srw->srw_state > thd_id && srw->srw_xlocker) { - xt_spinlock_unlock(&srw->srw_state_lock); - xt_critical_wait(); - xt_spinlock_lock(&srw->srw_state_lock); - } - xt_spinlock_unlock(&srw->srw_state_lock); - } + if (sxs->sxs_xlocked) { + /* I am waiting... */ + xt_atomic_inc2(&sxs->sxs_wait_count); + while (sxs->sxs_xlocked) + xt_yield(); + xt_atomic_dec2(&sxs->sxs_wait_count); } - /* There is no exclusive locker, so we have the read lock: */ - srw->x.srw_rlock[thd_id] = XT_HAVE_LOCK; #ifdef XT_THREAD_LOCK_INFO - xt_thread_lock_info_add_owner(&srw->srw_lock_info); + xt_thread_lock_info_add_owner(&sxs->sxs_lock_info); #endif - return OK; } -xtPublic xtBool xt_spinrwlock_unlock(XTSpinRWLockPtr srw, xtThreadID thd_id) +xtPublic xtBool xt_spinxslock_unlock(XTSpinXSLockPtr sxs, xtBool xlocked) { - if (srw->srw_xlocker == thd_id) { - /* I have an X lock. */ - ASSERT_NS(srw->srw_state == xt_thr_maximum_threads); - srw->srw_state = 0; - srw->srw_xlocker = 0; - xt_spinlock_unlock(&srw->srw_lock); - } - else { - /* I have a shared lock: */ - ASSERT_NS(srw->x.srw_rlock[thd_id] == XT_HAVE_LOCK); - ASSERT_NS(srw->srw_state != xt_thr_maximum_threads); - srw->x.srw_rlock[thd_id] = XT_NO_LOCK; - if (srw->srw_xlocker && srw->srw_state == thd_id) { - xt_spinlock_lock(&srw->srw_state_lock); - if (srw->srw_xlocker && srw->srw_state == thd_id) { - /* If the X locker is waiting for me, - * then allow him to continue. - */ - srw->srw_state = thd_id+1; - } - xt_spinlock_unlock(&srw->srw_state_lock); - } + if (xlocked) { +#ifdef DEBUG + sxs->sxs_locker = 0; +#endif + sxs->sxs_xlocked = 0; } + else + xt_atomic_dec2(&sxs->sxs_rlock_count); #ifdef XT_THREAD_LOCK_INFO - xt_thread_lock_info_release_owner(&srw->srw_lock_info); + xt_thread_lock_info_release_owner(&sxs->sxs_lock_info); #endif - return OK; } @@ -1550,194 +1512,159 @@ xtPublic xtBool xt_spinrwlock_unlock(XTS */ #ifdef XT_THREAD_LOCK_INFO -xtPublic void xt_fastrwlock_init(struct XTThread *self, XTFastRWLockPtr frw, const char *n) +xtPublic void xt_xsmutex_init(struct XTThread *self, XTXSMutexLockPtr xsm, const char *name) #else -xtPublic void xt_fastrwlock_init(struct XTThread *self, XTFastRWLockPtr frw) +xtPublic void xt_xsmutex_init(struct XTThread *self, XTXSMutexLockPtr xsm) #endif { - xt_fastlock_init_with_autoname(self, &frw->frw_lock); - frw->frw_xlocker = NULL; - xt_spinlock_init_with_autoname(self, &frw->frw_state_lock); - frw->frw_state = 0; - frw->frw_read_waiters = 0; - /* Must be aligned! */ - ASSERT(xt_thr_maximum_threads == xt_align_size(xt_thr_maximum_threads, XT_XS_LOCK_ALIGN)); - frw->x.frw_rlock = (xtWord1 *) xt_calloc(self, xt_thr_maximum_threads); + xt_init_mutex_with_autoname(self, &xsm->xsm_lock); + xt_init_cond(self, &xsm->xsm_cond); + xt_init_cond(self, &xsm->xsm_cond_2); + xsm->xsm_xlocker = 0; + xsm->xsm_rlock_count = 0; + xsm->xsm_wait_count = 0; +#ifdef DEBUG + xsm->xsm_locker = 0; +#endif #ifdef XT_THREAD_LOCK_INFO - frw->frw_name = n; - xt_thread_lock_info_init(&frw->frw_lock_info, frw); + xsm->xsm_name = name; + xt_thread_lock_info_init(&xsm->xsm_lock_info, xsm); #endif } -xtPublic void xt_fastrwlock_free(struct XTThread *self, XTFastRWLockPtr frw) +xtPublic void xt_xsmutex_free(struct XTThread *XT_UNUSED(self), XTXSMutexLockPtr xsm) { - if (frw->x.frw_rlock) - xt_free(self, (void *) frw->x.frw_rlock); - xt_fastlock_free(self, &frw->frw_lock); - xt_spinlock_free(self, &frw->frw_state_lock); + xt_free_mutex(&xsm->xsm_lock); + xt_free_cond(&xsm->xsm_cond); + xt_free_cond(&xsm->xsm_cond_2); #ifdef XT_THREAD_LOCK_INFO - xt_thread_lock_info_free(&frw->frw_lock_info); + xt_thread_lock_info_free(&xsm->xsm_lock_info); #endif } -xtPublic xtBool xt_fastrwlock_xlock(XTFastRWLockPtr frw, struct XTThread *thread) +xtPublic xtBool xt_xsmutex_xlock(XTXSMutexLockPtr xsm, xtThreadID thd_id) { - xt_fastlock_lock(&frw->frw_lock, thread); - ASSERT_NS(frw->x.frw_rlock[thread->t_id] == XT_NO_LOCK); - - xt_spinlock_lock(&frw->frw_state_lock); - - /* Set the state before xlocker (dirty read!) */ - frw->frw_state = 0; + xt_lock_mutex_ns(&xsm->xsm_lock); - /* I am the locker: */ - frw->frw_xlocker = thread; - - /* Wait for all the read lockers: */ - while (frw->frw_state < xt_thr_current_max_threads) { - while (frw->x.frw_rlock[frw->frw_state]) { - xt_lock_thread(thread); - xt_spinlock_unlock(&frw->frw_state_lock); - /* Wait for this reader. We rely on the reader to free - * us from this wait! */ - if (!xt_wait_thread(thread)) { - xt_unlock_thread(thread); - frw->frw_state = 0; - frw->frw_xlocker = NULL; - xt_fastlock_unlock(&frw->frw_lock, thread); - return FAILED; - } - xt_unlock_thread(thread); - xt_spinlock_lock(&frw->frw_state_lock); + /* Wait for exclusive locker: */ + while (xsm->xsm_xlocker) { + if (!xt_timed_wait_cond_ns(&xsm->xsm_cond, &xsm->xsm_lock, 10000)) { + xt_unlock_mutex_ns(&xsm->xsm_lock); + return FAILED; } - /* State can be incremented in parallel by a reader - * thread! - */ - frw->frw_state++; } - /* I have waited for all: */ - frw->frw_state = xt_thr_maximum_threads; - - xt_spinlock_unlock(&frw->frw_state_lock); + /* GOTCHA: You would think this is not necessary... + * But is does not always work, if a normal insert is used. + * The reason is, I guess, on MMP the assignment is not + * always immediately visible to other processors, because they + * have old versions of this variable in there cache. + * + * But this is required, because the locking mechanism is based + * on: + * Locker: sets xlocker, tests rlock_count + * Reader: incs rlock_count, tests xlocker + * + * The test, in both cases, may not read stale values. + * volatile does not help, because this just turns compiler + * optimisations off. + */ + xt_atomic_set4(&xsm->xsm_xlocker, thd_id); + + /* Wait for all the reader to wait! */ + while (xsm->xsm_wait_count < xsm->xsm_rlock_count) { + /* {RACE-WR_MUTEX} Here as well: */ + if (!xt_timed_wait_cond_ns(&xsm->xsm_cond, &xsm->xsm_lock, 100)) { + xsm->xsm_xlocker = 0; + xt_unlock_mutex_ns(&xsm->xsm_lock); + return FAILED; + } + } #ifdef XT_THREAD_LOCK_INFO - xt_thread_lock_info_add_owner(&frw->frw_lock_info); + xt_thread_lock_info_add_owner(&xsm->xsm_lock_info); #endif - return OK; } -xtPublic xtBool xt_fastrwlock_slock(XTFastRWLockPtr frw, struct XTThread *thread) +xtPublic xtBool xt_xsmutex_slock(XTXSMutexLockPtr xsm, xtThreadID XT_UNUSED(thd_id)) { - xtThreadID thd_id = thread->t_id; + xt_atomic_inc2(&xsm->xsm_rlock_count); - ASSERT_NS(frw->x.frw_rlock[thd_id] == XT_NO_LOCK); - frw->x.frw_rlock[thd_id] = XT_WANT_LOCK; /* Check if there could be an X locker: */ - if (frw->frw_xlocker) { - /* There is an X locker. - * If frw_state < thd_id then the X locker will wait for me. - * So I should not wait! - */ - if (frw->frw_state >= thd_id) { - /* If frw->frw_state >= thd_id, then the locker may have, or - * has already checked me, and I will have to wait. - * - * Otherwise, frw_state <= thd_id, which means the - * X locker has not checked me, and will still wait for me (or - * is already waiting for me). In this case, I will have to - * take the mutex to make sure exactly how far he - * is with the checking. - */ - xt_spinlock_lock(&frw->frw_state_lock); - frw->frw_read_waiters++; - frw->x.frw_rlock[thd_id] = XT_WAITING; - while (frw->frw_state > thd_id && frw->frw_xlocker) { - xt_lock_thread(thread); - xt_spinlock_unlock(&frw->frw_state_lock); - if (!xt_wait_thread(thread)) { - xt_unlock_thread(thread); - xt_spinlock_lock(&frw->frw_state_lock); - frw->frw_read_waiters--; - frw->x.frw_rlock[thd_id] = XT_NO_LOCK; - xt_spinlock_unlock(&frw->frw_state_lock); - return FAILED; - } - xt_unlock_thread(thread); - xt_spinlock_lock(&frw->frw_state_lock); + if (xsm->xsm_xlocker) { + /* I am waiting... */ + xt_lock_mutex_ns(&xsm->xsm_lock); + xsm->xsm_wait_count++; + /* Wake up the xlocker: */ + if (xsm->xsm_xlocker && xsm->xsm_wait_count == xsm->xsm_rlock_count) { + if (!xt_broadcast_cond_ns(&xsm->xsm_cond)) { + xsm->xsm_wait_count--; + xt_unlock_mutex_ns(&xsm->xsm_lock); + return FAILED; } - frw->x.frw_rlock[thd_id] = XT_HAVE_LOCK; - frw->frw_read_waiters--; - xt_spinlock_unlock(&frw->frw_state_lock); - return OK; } + while (xsm->xsm_xlocker) { + if (!xt_timed_wait_cond_ns(&xsm->xsm_cond_2, &xsm->xsm_lock, 10000)) { + xsm->xsm_wait_count--; + xt_unlock_mutex_ns(&xsm->xsm_lock); + return FAILED; + } + } + xsm->xsm_wait_count--; + xt_unlock_mutex_ns(&xsm->xsm_lock); } - /* There is no exclusive locker, so we have the read lock: */ - frw->x.frw_rlock[thd_id] = XT_HAVE_LOCK; #ifdef XT_THREAD_LOCK_INFO - xt_thread_lock_info_add_owner(&frw->frw_lock_info); + xt_thread_lock_info_add_owner(&xsm->xsm_lock_info); #endif - return OK; } -xtPublic xtBool xt_fastrwlock_unlock(XTFastRWLockPtr frw, struct XTThread *thread) +xtPublic xtBool xt_xsmutex_unlock(XTXSMutexLockPtr xsm, xtThreadID thd_id) { - xtThreadID thd_id = thread->t_id; - - if (frw->frw_xlocker == thread) { - /* I have an X lock. */ - ASSERT_NS(frw->frw_state == xt_thr_maximum_threads); - frw->frw_state = 0; - frw->frw_xlocker = NULL; - - /* Wake up all read waiters: */ - if (frw->frw_read_waiters) { - xt_spinlock_lock(&frw->frw_state_lock); - if (frw->frw_read_waiters) { - XTThreadPtr target; - - for (u_int i=0; i<xt_thr_current_max_threads; i++) { - if (frw->x.frw_rlock[i] == XT_WAITING) { - if ((target = xt_thr_array[i])) { - xt_lock_thread(target); - xt_signal_thread(target); - xt_unlock_thread(target); - } - } - } + if (xsm->xsm_xlocker == thd_id) { + xsm->xsm_xlocker = 0; + if (xsm->xsm_wait_count) { + if (!xt_broadcast_cond_ns(&xsm->xsm_cond_2)) { + xt_unlock_mutex_ns(&xsm->xsm_lock); + return FAILED; } - xt_spinlock_unlock(&frw->frw_state_lock); } - xt_fastlock_unlock(&frw->frw_lock, thread); + else { + /* Wake up any other X or shared lockers: */ + if (!xt_broadcast_cond_ns(&xsm->xsm_cond)) { + xt_unlock_mutex_ns(&xsm->xsm_lock); + return FAILED; + } + } + xt_unlock_mutex_ns(&xsm->xsm_lock); } else { - /* I have a shared lock: */ - ASSERT_NS(frw->x.frw_rlock[thd_id] == XT_HAVE_LOCK); - ASSERT_NS(frw->frw_state != xt_thr_maximum_threads); - frw->x.frw_rlock[thd_id] = XT_NO_LOCK; - if (frw->frw_xlocker && frw->frw_state == thd_id) { - xt_spinlock_lock(&frw->frw_state_lock); - if (frw->frw_xlocker && frw->frw_state == thd_id) { + /* Taking the advice from {RACE-WR_MUTEX} I do the decrement + * after I have a lock! + */ + if (xsm->xsm_xlocker) { + xt_lock_mutex_ns(&xsm->xsm_lock); + xt_atomic_dec2(&xsm->xsm_rlock_count); + if (xsm->xsm_xlocker && xsm->xsm_wait_count == xsm->xsm_rlock_count) { /* If the X locker is waiting for me, * then allow him to continue. */ - frw->frw_state = thd_id+1; - /* Wake him up: */ - xt_lock_thread(frw->frw_xlocker); - xt_signal_thread(frw->frw_xlocker); - xt_unlock_thread(frw->frw_xlocker); + if (!xt_broadcast_cond_ns(&xsm->xsm_cond)) { + xt_unlock_mutex_ns(&xsm->xsm_lock); + return FAILED; + } } - xt_spinlock_unlock(&frw->frw_state_lock); + xt_unlock_mutex_ns(&xsm->xsm_lock); } + else + xt_atomic_dec2(&xsm->xsm_rlock_count); } #ifdef XT_THREAD_LOCK_INFO - xt_thread_lock_info_release_owner(&frw->frw_lock_info); + xt_thread_lock_info_release_owner(&xsm->xsm_lock_info); #endif - return OK; } @@ -1747,9 +1674,9 @@ xtPublic xtBool xt_fastrwlock_unlock(XTF */ #ifdef XT_THREAD_LOCK_INFO -xtPublic void xt_atomicrwlock_init(struct XTThread XT_UNUSED(*self), XTAtomicRWLockPtr arw, const char *n) +xtPublic void xt_atomicrwlock_init(struct XTThread *XT_UNUSED(self), XTAtomicRWLockPtr arw, const char *n) #else -xtPublic void xt_atomicrwlock_init(struct XTThread XT_UNUSED(*self), XTAtomicRWLockPtr arw) +xtPublic void xt_atomicrwlock_init(struct XTThread *XT_UNUSED(self), XTAtomicRWLockPtr arw) #endif { arw->arw_reader_count = 0; @@ -1760,14 +1687,18 @@ xtPublic void xt_atomicrwlock_init(struc #endif } +#ifdef XT_THREAD_LOCK_INFO +xtPublic void xt_atomicrwlock_free(struct XTThread *, XTAtomicRWLockPtr arw) +#else xtPublic void xt_atomicrwlock_free(struct XTThread *, XTAtomicRWLockPtr XT_UNUSED(arw)) +#endif { #ifdef XT_THREAD_LOCK_INFO xt_thread_lock_info_free(&arw->arw_lock_info); #endif } -xtPublic xtBool xt_atomicrwlock_xlock(XTAtomicRWLockPtr arw, xtThreadID XT_UNUSED(thr_id)) +xtPublic xtBool xt_atomicrwlock_xlock(XTAtomicRWLockPtr arw, xtThreadID XT_NDEBUG_UNUSED(thr_id)) { register xtWord2 set; @@ -1819,16 +1750,118 @@ xtPublic xtBool xt_atomicrwlock_slock(XT xtPublic xtBool xt_atomicrwlock_unlock(XTAtomicRWLockPtr arw, xtBool xlocked) { - if (xlocked) + if (xlocked) { +#ifdef DEBUG + arw->arw_locker = 0; +#endif arw->arw_xlock_set = 0; + } else xt_atomic_dec2(&arw->arw_reader_count); #ifdef XT_THREAD_LOCK_INFO xt_thread_lock_info_release_owner(&arw->arw_lock_info); #endif + + return OK; +} + +/* + * ----------------------------------------------------------------------- + * "SKEW" ATOMITC READ/WRITE LOCK (BASED ON ATOMIC OPERATIONS) + * + * This lock type favors writers. It only works if the proportion of readers + * to writer is high. + */ + +#ifdef XT_THREAD_LOCK_INFO +xtPublic void xt_skewrwlock_init(struct XTThread *XT_UNUSED(self), XTSkewRWLockPtr srw, const char *n) +#else +xtPublic void xt_skewrwlock_init(struct XTThread *XT_UNUSED(self), XTSkewRWLockPtr srw) +#endif +{ + srw->srw_reader_count = 0; + srw->srw_xlock_set = 0; +#ifdef XT_THREAD_LOCK_INFO + srw->srw_name = n; + xt_thread_lock_info_init(&srw->srw_lock_info, srw); +#endif +} + +#ifdef XT_THREAD_LOCK_INFO +xtPublic void xt_skewrwlock_free(struct XTThread *, XTSkewRWLockPtr srw) +#else +xtPublic void xt_skewrwlock_free(struct XTThread *, XTSkewRWLockPtr XT_UNUSED(srw)) +#endif +{ +#ifdef XT_THREAD_LOCK_INFO + xt_thread_lock_info_free(&srw->srw_lock_info); +#endif +} + +xtPublic xtBool xt_skewrwlock_xlock(XTSkewRWLockPtr srw, xtThreadID XT_NDEBUG_UNUSED(thr_id)) +{ + register xtWord2 set; + + /* First get an exclusive lock: */ + for (;;) { + set = xt_atomic_tas2(&srw->srw_xlock_set, 1); + if (!set) + break; + xt_yield(); + } + + /* Wait for the remaining readers: */ + while (srw->srw_reader_count) + xt_yield(); + +#ifdef DEBUG + srw->srw_locker = thr_id; +#endif + +#ifdef XT_THREAD_LOCK_INFO + xt_thread_lock_info_add_owner(&srw->srw_lock_info); +#endif + return OK; +} + +xtPublic xtBool xt_skewrwlock_slock(XTSkewRWLockPtr srw) +{ + /* Wait for an exclusive lock: */ + retry: + for (;;) { + if (!srw->srw_xlock_set) + break; + xt_yield(); + } + + /* Add a reader: */ + xt_atomic_inc2(&srw->srw_reader_count); + + /* Check for xlock again: */ + if (srw->srw_xlock_set) { + xt_atomic_dec2(&srw->srw_reader_count); + goto retry; + } + +#ifdef XT_THREAD_LOCK_INFO + xt_thread_lock_info_add_owner(&srw->srw_lock_info); +#endif + return OK; +} + +xtPublic xtBool xt_skewrwlock_unlock(XTSkewRWLockPtr srw, xtBool xlocked) +{ + if (xlocked) + srw->srw_xlock_set = 0; + else + xt_atomic_dec2(&srw->srw_reader_count); + +#ifdef XT_THREAD_LOCK_INFO + xt_thread_lock_info_release_owner(&srw->srw_lock_info); +#endif #ifdef DEBUG - arw->arw_locker = 0; + srw->srw_locker = 0; #endif return OK; @@ -1844,15 +1877,17 @@ xtPublic xtBool xt_atomicrwlock_unlock(X #define JOB_PRINT 3 #define JOB_INCREMENT 4 #define JOB_SNOOZE 5 +#define JOB_DOUBLE_INC 6 #define LOCK_PTHREAD_RW 1 #define LOCK_PTHREAD_MUTEX 2 -#define LOCK_FASTRW 3 +#define LOCK_RWMUTEX 3 #define LOCK_SPINLOCK 4 #define LOCK_FASTLOCK 5 -#define LOCK_SPINRWLOCK 6 -#define LOCK_FASTRWLOCK 7 +#define LOCK_SPINXSLOCK 6 +#define LOCK_XSMUTEX 7 #define LOCK_ATOMICRWLOCK 8 +#define LOCK_SKEWRWLOCK 9 typedef struct XSLockTest { u_int xs_interations; @@ -1864,18 +1899,19 @@ typedef struct XSLockTest { XTSpinLockRec xs_spinlock; xt_mutex_type xs_mutex; XTFastLockRec xs_fastlock; - XTSpinRWLockRec xs_spinrwlock; - XTFastRWLockRec xs_fastrwlock; + XTSpinXSLockRec xs_spinrwlock; + XTXSMutexRec xs_fastrwlock; XTAtomicRWLockRec xs_atomicrwlock; + XTSkewRWLockRec xs_skewrwlock; int xs_progress; xtWord4 xs_inc; } XSLockTestRec, *XSLockTestPtr; -static void lck_free_thread_data(XTThreadPtr self __attribute__((unused)), void *data __attribute__((unused))) +static void lck_free_thread_data(XTThreadPtr XT_UNUSED(self), void *XT_UNUSED(data)) { } -static void lck_do_job(XTThreadPtr self, int job, XSLockTestPtr data) +static void lck_do_job(XTThreadPtr self, int job, XSLockTestPtr data, xtBool reader) { char b1[2048], b2[2048]; @@ -1900,6 +1936,16 @@ static void lck_do_job(XTThreadPtr self, xt_sleep_milli_second(10); data->xs_inc++; break; + case JOB_DOUBLE_INC: + if (reader) { + if ((data->xs_inc & 1) != 0) + printf("Noooo!\n"); + } + else { + data->xs_inc++; + data->xs_inc++; + } + break; } } @@ -1929,29 +1975,34 @@ static void *lck_run_reader(XTThreadPtr printf("- %s %d\n", self->t_name, i+1); if (data->xs_which_lock == LOCK_PTHREAD_RW) { xt_slock_rwlock_ns(&data->xs_plock); - lck_do_job(self, data->xs_which_job, data); + lck_do_job(self, data->xs_which_job, data, TRUE); xt_unlock_rwlock_ns(&data->xs_plock); } - else if (data->xs_which_lock == LOCK_FASTRW) { + else if (data->xs_which_lock == LOCK_RWMUTEX) { xt_rwmutex_slock(&data->xs_lock, self->t_id); - lck_do_job(self, data->xs_which_job, data); + lck_do_job(self, data->xs_which_job, data, TRUE); xt_rwmutex_unlock(&data->xs_lock, self->t_id); } - else if (data->xs_which_lock == LOCK_SPINRWLOCK) { - xt_spinrwlock_slock(&data->xs_spinrwlock, self->t_id); - lck_do_job(self, data->xs_which_job, data); - xt_spinrwlock_unlock(&data->xs_spinrwlock, self->t_id); - } - else if (data->xs_which_lock == LOCK_FASTRWLOCK) { - xt_fastrwlock_slock(&data->xs_fastrwlock, self); - lck_do_job(self, data->xs_which_job, data); - xt_fastrwlock_unlock(&data->xs_fastrwlock, self); + else if (data->xs_which_lock == LOCK_SPINXSLOCK) { + xt_spinxslock_slock(&data->xs_spinrwlock); + lck_do_job(self, data->xs_which_job, data, TRUE); + xt_spinxslock_unlock(&data->xs_spinrwlock, FALSE); + } + else if (data->xs_which_lock == LOCK_XSMUTEX) { + xt_xsmutex_slock(&data->xs_fastrwlock, self->t_id); + lck_do_job(self, data->xs_which_job, data, TRUE); + xt_xsmutex_unlock(&data->xs_fastrwlock, self->t_id); } else if (data->xs_which_lock == LOCK_ATOMICRWLOCK) { xt_atomicrwlock_slock(&data->xs_atomicrwlock); - lck_do_job(self, data->xs_which_job, data); + lck_do_job(self, data->xs_which_job, data, TRUE); xt_atomicrwlock_unlock(&data->xs_atomicrwlock, FALSE); } + else if (data->xs_which_lock == LOCK_SKEWRWLOCK) { + xt_skewrwlock_slock(&data->xs_skewrwlock); + lck_do_job(self, data->xs_which_job, data, TRUE); + xt_skewrwlock_unlock(&data->xs_skewrwlock, FALSE); + } else ASSERT(FALSE); } @@ -1971,29 +2022,34 @@ static void *lck_run_writer(XTThreadPtr printf("- %s %d\n", self->t_name, i+1); if (data->xs_which_lock == LOCK_PTHREAD_RW) { xt_xlock_rwlock_ns(&data->xs_plock); - lck_do_job(self, data->xs_which_job, data); + lck_do_job(self, data->xs_which_job, data, FALSE); xt_unlock_rwlock_ns(&data->xs_plock); } - else if (data->xs_which_lock == LOCK_FASTRW) { + else if (data->xs_which_lock == LOCK_RWMUTEX) { xt_rwmutex_xlock(&data->xs_lock, self->t_id); - lck_do_job(self, data->xs_which_job, data); + lck_do_job(self, data->xs_which_job, data, FALSE); xt_rwmutex_unlock(&data->xs_lock, self->t_id); } - else if (data->xs_which_lock == LOCK_SPINRWLOCK) { - xt_spinrwlock_xlock(&data->xs_spinrwlock, self->t_id); - lck_do_job(self, data->xs_which_job, data); - xt_spinrwlock_unlock(&data->xs_spinrwlock, self->t_id); - } - else if (data->xs_which_lock == LOCK_FASTRWLOCK) { - xt_fastrwlock_xlock(&data->xs_fastrwlock, self); - lck_do_job(self, data->xs_which_job, data); - xt_fastrwlock_unlock(&data->xs_fastrwlock, self); + else if (data->xs_which_lock == LOCK_SPINXSLOCK) { + xt_spinxslock_xlock(&data->xs_spinrwlock, self->t_id); + lck_do_job(self, data->xs_which_job, data, FALSE); + xt_spinxslock_unlock(&data->xs_spinrwlock, TRUE); + } + else if (data->xs_which_lock == LOCK_XSMUTEX) { + xt_xsmutex_xlock(&data->xs_fastrwlock, self->t_id); + lck_do_job(self, data->xs_which_job, data, FALSE); + xt_xsmutex_unlock(&data->xs_fastrwlock, self->t_id); } else if (data->xs_which_lock == LOCK_ATOMICRWLOCK) { xt_atomicrwlock_xlock(&data->xs_atomicrwlock, self->t_id); - lck_do_job(self, data->xs_which_job, data); + lck_do_job(self, data->xs_which_job, data, FALSE); xt_atomicrwlock_unlock(&data->xs_atomicrwlock, TRUE); } + else if (data->xs_which_lock == LOCK_SKEWRWLOCK) { + xt_skewrwlock_xlock(&data->xs_skewrwlock, self->t_id); + lck_do_job(self, data->xs_which_job, data, FALSE); + xt_skewrwlock_unlock(&data->xs_skewrwlock, TRUE); + } else ASSERT(FALSE); } @@ -2011,7 +2067,7 @@ static void lck_print_test(XSLockTestRec case LOCK_PTHREAD_MUTEX: printf("pthread mutex"); break; - case LOCK_FASTRW: + case LOCK_RWMUTEX: printf("fast read/write mutex"); break; case LOCK_SPINLOCK: @@ -2020,15 +2076,18 @@ static void lck_print_test(XSLockTestRec case LOCK_FASTLOCK: printf("fast mutex"); break; - case LOCK_SPINRWLOCK: + case LOCK_SPINXSLOCK: printf("spin read/write lock"); break; - case LOCK_FASTRWLOCK: - printf("fast read/write lock"); + case LOCK_XSMUTEX: + printf("fast x/s mutex"); break; case LOCK_ATOMICRWLOCK: printf("atomic read/write lock"); break; + case LOCK_SKEWRWLOCK: + printf("skew read/write lock"); + break; } switch (data->xs_which_job) { @@ -2063,17 +2122,17 @@ static void *lck_run_mutex_locker(XTThre printf("- %s %d\n", self->t_name, i+1); if (data->xs_which_lock == LOCK_PTHREAD_MUTEX) { xt_lock_mutex_ns(&data->xs_mutex); - lck_do_job(self, data->xs_which_job, data); + lck_do_job(self, data->xs_which_job, data, FALSE); xt_unlock_mutex_ns(&data->xs_mutex); } else if (data->xs_which_lock == LOCK_SPINLOCK) { xt_spinlock_lock(&data->xs_spinlock); - lck_do_job(self, data->xs_which_job, data); + lck_do_job(self, data->xs_which_job, data, FALSE); xt_spinlock_unlock(&data->xs_spinlock); } else if (data->xs_which_lock == LOCK_FASTLOCK) { xt_fastlock_lock(&data->xs_fastlock, self); - lck_do_job(self, data->xs_which_job, data); + lck_do_job(self, data->xs_which_job, data, FALSE); xt_fastlock_unlock(&data->xs_fastlock, self); } else @@ -2164,15 +2223,19 @@ xtPublic void xt_unit_test_read_write_lo memset(&data, 0, sizeof(data)); printf("TEST: xt_unit_test_read_write_locks\n"); + printf("size of XTXSMutexRec = %d\n", (int) sizeof(XTXSMutexRec)); + printf("size of pthread_cond_t = %d\n", (int) sizeof(pthread_cond_t)); + printf("size of pthread_mutex_t = %d\n", (int) sizeof(pthread_mutex_t)); xt_rwmutex_init_with_autoname(self, &data.xs_lock); xt_init_rwlock_with_autoname(self, &data.xs_plock); - xt_spinrwlock_init_with_autoname(self, &data.xs_spinrwlock); - xt_fastrwlock_init_with_autoname(self, &data.xs_fastrwlock); + xt_spinxslock_init_with_autoname(self, &data.xs_spinrwlock); + xt_xsmutex_init_with_autoname(self, &data.xs_fastrwlock); xt_atomicrwlock_init_with_autoname(self, &data.xs_atomicrwlock); + xt_skewrwlock_init_with_autoname(self, &data.xs_skewrwlock); /** data.xs_interations = 10; - data.xs_which_lock = LOCK_FASTRW; // LOCK_PTHREAD_RW, LOCK_FASTRW, LOCK_SPINRWLOCK, LOCK_FASTRWLOCK + data.xs_which_lock = LOCK_RWMUTEX; // LOCK_PTHREAD_RW, LOCK_RWMUTEX, LOCK_SPINXSLOCK, LOCK_XSMUTEX data.xs_which_job = JOB_PRINT; data.xs_debug_print = TRUE; data.xs_progress = 0; @@ -2184,7 +2247,7 @@ xtPublic void xt_unit_test_read_write_lo /** data.xs_interations = 4000; - data.xs_which_lock = LOCK_FASTRW; // LOCK_PTHREAD_RW, LOCK_FASTRW, LOCK_SPINRWLOCK, LOCK_FASTRWLOCK + data.xs_which_lock = LOCK_RWMUTEX; // LOCK_PTHREAD_RW, LOCK_RWMUTEX, LOCK_SPINXSLOCK, LOCK_XSMUTEX data.xs_which_job = JOB_SLEEP; data.xs_debug_print = TRUE; data.xs_progress = 200; @@ -2194,37 +2257,52 @@ xtPublic void xt_unit_test_read_write_lo lck_reader_writer_test(self, &data, 4, 2); **/ + // LOCK_PTHREAD_RW, LOCK_RWMUTEX, LOCK_SPINXSLOCK, LOCK_XSMUTEX, LOCK_ATOMICRWLOCK, LOCK_SKEWRWLOCK /**/ - data.xs_interations = 1000000; - data.xs_which_lock = LOCK_FASTRW; // LOCK_PTHREAD_RW, LOCK_FASTRW, LOCK_SPINRWLOCK, LOCK_FASTRWLOCK, LOCK_ATOMICRWLOCK - data.xs_which_job = JOB_INCREMENT; + data.xs_interations = 100000; + data.xs_which_lock = LOCK_XSMUTEX; + data.xs_which_job = JOB_DOUBLE_INC; // JOB_INCREMENT, JOB_DOUBLE_INC data.xs_debug_print = FALSE; data.xs_progress = 0; lck_reader_writer_test(self, &data, 10, 0); + data.xs_which_lock = LOCK_XSMUTEX; + lck_reader_writer_test(self, &data, 10, 0); + //lck_reader_writer_test(self, &data, 0, 5); + //lck_reader_writer_test(self, &data, 10, 0); + //lck_reader_writer_test(self, &data, 10, 5); /**/ - /** + /**/ data.xs_interations = 10000; - data.xs_which_lock = LOCK_FASTRW; // LOCK_PTHREAD_RW, LOCK_FASTRW, LOCK_SPINRWLOCK, LOCK_FASTRWLOCK + data.xs_which_lock = LOCK_XSMUTEX; data.xs_which_job = JOB_MEMCPY; data.xs_debug_print = FALSE; data.xs_progress = 0; - lck_reader_writer_test(self, &data, 10, 5); - **/ + lck_reader_writer_test(self, &data, 10, 0); + data.xs_which_lock = LOCK_XSMUTEX; + lck_reader_writer_test(self, &data, 10, 0); + //lck_reader_writer_test(self, &data, 0, 5); + //lck_reader_writer_test(self, &data, 10, 0); + //lck_reader_writer_test(self, &data, 10, 5); + /**/ - /** + /**/ data.xs_interations = 1000; - data.xs_which_lock = LOCK_FASTRW; // LOCK_PTHREAD_RW, LOCK_FASTRW, LOCK_SPINRWLOCK, LOCK_FASTRWLOCK - data.xs_which_job = JOB_SLEEP; + data.xs_which_lock = LOCK_XSMUTEX; + data.xs_which_job = JOB_SLEEP; // JOB_SLEEP, JOB_SNOOZE data.xs_debug_print = FALSE; data.xs_progress = 0; - lck_reader_writer_test(self, &data, 10, 5); - **/ + lck_reader_writer_test(self, &data, 10, 0); + data.xs_which_lock = LOCK_XSMUTEX; + lck_reader_writer_test(self, &data, 10, 0); + /**/ xt_rwmutex_free(self, &data.xs_lock); xt_free_rwlock(&data.xs_plock); - xt_spinrwlock_free(self, &data.xs_spinrwlock); - xt_fastrwlock_free(self, &data.xs_fastrwlock); + xt_spinxslock_free(self, &data.xs_spinrwlock); + xt_xsmutex_free(self, &data.xs_fastrwlock); + xt_atomicrwlock_free(self, &data.xs_atomicrwlock); + xt_skewrwlock_free(self, &data.xs_skewrwlock); } xtPublic void xt_unit_test_mutex_locks(XTThreadPtr self) === modified file 'storage/pbxt/src/lock_xt.h' --- a/storage/pbxt/src/lock_xt.h 2009-05-09 04:01:53 +0000 +++ b/storage/pbxt/src/lock_xt.h 2009-08-17 11:12:36 +0000 @@ -36,96 +36,16 @@ struct XTOpenTable; struct XTXactData; struct XTTable; -/* Possibilities are 2 = align 4 or 2 = align 8 */ -#define XT_XS_LOCK_SHIFT 2 -#define XT_XS_LOCK_ALIGN (1 << XT_XS_LOCK_SHIFT) - -/* This lock is fast for reads but slow for writes. - * Use this lock in situations where you have 99% reads, - * and then some potentially long writes. - */ -typedef struct XTRWMutex { -#ifdef DEBUG - struct XTThread *xs_lock_thread; - u_int xs_inited; -#endif -#ifdef XT_THREAD_LOCK_INFO - XTThreadLockInfoRec xs_lock_info; - const char *xs_name; -#endif - xt_mutex_type xs_lock; - xt_cond_type xs_cond; - volatile xtWord4 xs_state; - volatile xtThreadID xs_xlocker; - union { -#if XT_XS_LOCK_ALIGN == 4 - volatile xtWord4 *xs_rlock_align; -#else - volatile xtWord8 *xs_rlock_align; -#endif - volatile xtWord1 *xs_rlock; - } x; -} XTRWMutexRec, *XTRWMutexPtr; - -#ifdef XT_THREAD_LOCK_INFO -#define xt_rwmutex_init_with_autoname(a,b) xt_rwmutex_init(a,b,LOCKLIST_ARG_SUFFIX(b)) -void xt_rwmutex_init(struct XTThread *self, XTRWMutexPtr xsl, const char *name); -#else -#define xt_rwmutex_init_with_autoname(a,b) xt_rwmutex_init(a,b) -void xt_rwmutex_init(struct XTThread *self, XTRWMutexPtr xsl); -#endif -void xt_rwmutex_free(struct XTThread *self, XTRWMutexPtr xsl); -xtBool xt_rwmutex_xlock(XTRWMutexPtr xsl, xtThreadID thd_id); -xtBool xt_rwmutex_slock(XTRWMutexPtr xsl, xtThreadID thd_id); -xtBool xt_rwmutex_unlock(XTRWMutexPtr xsl, xtThreadID thd_id); - -#ifdef XT_WIN -#define XT_SPL_WIN32_ASM -#else -#if defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__)) -#define XT_SPL_GNUC_X86 -#else -#define XT_SPL_DEFAULT -#endif -#endif - -#ifdef XT_SOLARIS -/* Use Sun atomic operations library - * http://docs.sun.com/app/docs/doc/816-5168/atomic-ops-3c?a=view - */ -#define XT_SPL_SOLARIS_LIB -#endif - -#ifdef XT_SPL_SOLARIS_LIB +#ifdef XT_ATOMIC_SOLARIS_LIB #include <atomic.h> #endif -typedef struct XTSpinLock { - volatile xtWord4 spl_lock; -#ifdef XT_SPL_DEFAULT - xt_mutex_type spl_mutex; -#endif -#ifdef DEBUG - struct XTThread *spl_locker; -#endif -#ifdef XT_THREAD_LOCK_INFO - XTThreadLockInfoRec spl_lock_info; - const char *spl_name; -#endif -} XTSpinLockRec, *XTSpinLockPtr; +void xt_log_atomic_error_and_abort(c_char *func, c_char *file, u_int line); -#ifdef XT_THREAD_LOCK_INFO -#define xt_spinlock_init_with_autoname(a,b) xt_spinlock_init(a,b,LOCKLIST_ARG_SUFFIX(b)) -void xt_spinlock_init(struct XTThread *self, XTSpinLockPtr sp, const char *name); -#else -#define xt_spinlock_init_with_autoname(a,b) xt_spinlock_init(a,b) -void xt_spinlock_init(struct XTThread *self, XTSpinLockPtr sp); -#endif -void xt_spinlock_free(struct XTThread *self, XTSpinLockPtr sp); -xtBool xt_spinlock_spin(XTSpinLockPtr spl); -#ifdef DEBUG -void xt_spinlock_set_thread(XTSpinLockPtr spl); -#endif +/* + * ----------------------------------------------------------------------- + * ATOMIC OPERATIONS + */ /* * This macro is to remind me where it was safe @@ -137,37 +57,38 @@ void xt_spinlock_set_thread(XTSpinLockPt * is written atomically. * But the operations themselves are not atomic! */ -inline void xt_flushed_inc1(volatile xtWord1 *mptr) +inline void xt_atomic_inc1(volatile xtWord1 *mptr) { -#ifdef XT_SPL_WIN32_ASM +#ifdef XT_ATOMIC_WIN32_X86 __asm MOV ECX, mptr __asm MOV DL, BYTE PTR [ECX] __asm INC DL __asm XCHG DL, BYTE PTR [ECX] -#elif defined(XT_SPL_GNUC_X86) +#elif defined(XT_ATOMIC_GNUC_X86) xtWord1 val; asm volatile ("movb %1,%0" : "=r" (val) : "m" (*mptr) : "memory"); val++; asm volatile ("xchgb %1,%0" : "=r" (val) : "m" (*mptr), "0" (val) : "memory"); -#elif defined(XT_SPL_SOLARIS_LIB) +#elif defined(XT_ATOMIC_SOLARIS_LIB) atomic_inc_8(mptr); #else *mptr++; + xt_log_atomic_error_and_abort(__FUNC__, __FILE__, __LINE__); #endif } -inline xtWord1 xt_flushed_dec1(volatile xtWord1 *mptr) +inline xtWord1 xt_atomic_dec1(volatile xtWord1 *mptr) { xtWord1 val; -#ifdef XT_SPL_WIN32_ASM +#ifdef XT_ATOMIC_WIN32_X86 __asm MOV ECX, mptr __asm MOV DL, BYTE PTR [ECX] __asm DEC DL __asm MOV val, DL __asm XCHG DL, BYTE PTR [ECX] -#elif defined(XT_SPL_GNUC_X86) +#elif defined(XT_ATOMIC_GNUC_X86) xtWord1 val2; asm volatile ("movb %1, %0" : "=r" (val) : "m" (*mptr) : "memory"); @@ -176,55 +97,58 @@ inline xtWord1 xt_flushed_dec1(volatile /* Should work, but compiler makes a mistake? * asm volatile ("xchgb %1, %0" : : "r" (val), "m" (*mptr) : "memory"); */ -#elif defined(XT_SPL_SOLARIS_LIB) +#elif defined(XT_ATOMIC_SOLARIS_LIB) val = atomic_dec_8_nv(mptr); #else val = --(*mptr); + xt_log_atomic_error_and_abort(__FUNC__, __FILE__, __LINE__); #endif return val; } inline void xt_atomic_inc2(volatile xtWord2 *mptr) { -#ifdef XT_SPL_WIN32_ASM +#ifdef XT_ATOMIC_WIN32_X86 __asm LOCK INC WORD PTR mptr -#elif defined(XT_SPL_GNUC_X86) - asm volatile ("lock; incw %0" : : "m" (*mptr) : "memory"); -#elif defined(__GNUC__) +#elif defined(XT_ATOMIC_GNUC_X86) + asm volatile ("lock; incw %0" : : "m" (*mptr) : "memory"); +#elif defined(XT_ATOMIC_GCC_OPS) __sync_fetch_and_add(mptr, 1); -#elif defined(XT_SPL_SOLARIS_LIB) +#elif defined(XT_ATOMIC_SOLARIS_LIB) atomic_inc_16_nv(mptr); #else (*mptr)++; + xt_log_atomic_error_and_abort(__FUNC__, __FILE__, __LINE__); #endif } inline void xt_atomic_dec2(volatile xtWord2 *mptr) { -#ifdef XT_SPL_WIN32_ASM +#ifdef XT_ATOMIC_WIN32_X86 __asm LOCK DEC WORD PTR mptr -#elif defined(XT_SPL_GNUC_X86) +#elif defined(XT_ATOMIC_GNUC_X86) asm volatile ("lock; decw %0" : : "m" (*mptr) : "memory"); -#elif defined(__GNUC__) +#elif defined(XT_ATOMIC_GCC_OPS) __sync_fetch_and_sub(mptr, 1); -#elif defined(XT_SPL_SOLARIS_LIB) +#elif defined(XT_ATOMIC_SOLARIS_LIB) atomic_dec_16_nv(mptr); #else --(*mptr); + xt_log_atomic_error_and_abort(__FUNC__, __FILE__, __LINE__); #endif } /* Atomic test and set 2 byte word! */ inline xtWord2 xt_atomic_tas2(volatile xtWord2 *mptr, xtWord2 val) { -#ifdef XT_SPL_WIN32_ASM +#ifdef XT_ATOMIC_WIN32_X86 __asm MOV ECX, mptr __asm MOV DX, val __asm XCHG DX, WORD PTR [ECX] __asm MOV val, DX -#elif defined(XT_SPL_GNUC_X86) +#elif defined(XT_ATOMIC_GNUC_X86) asm volatile ("xchgw %1,%0" : "=r" (val) : "m" (*mptr), "0" (val) : "memory"); -#elif defined(XT_SPL_SOLARIS_LIB) +#elif defined(XT_ATOMIC_SOLARIS_LIB) val = atomic_swap_16(mptr, val); #else /* Yikes! */ @@ -232,43 +156,80 @@ inline xtWord2 xt_atomic_tas2(volatile x val = *mptr; *mptr = nval; + xt_log_atomic_error_and_abort(__FUNC__, __FILE__, __LINE__); #endif return val; } inline void xt_atomic_set4(volatile xtWord4 *mptr, xtWord4 val) { -#ifdef XT_SPL_WIN32_ASM +#ifdef XT_ATOMIC_WIN32_X86 __asm MOV ECX, mptr __asm MOV EDX, val __asm XCHG EDX, DWORD PTR [ECX] //__asm MOV DWORD PTR [ECX], EDX -#elif defined(XT_SPL_GNUC_X86) +#elif defined(XT_ATOMIC_GNUC_X86) asm volatile ("xchgl %1,%0" : "=r" (val) : "m" (*mptr), "0" (val) : "memory"); //asm volatile ("movl %0,%1" : "=r" (val) : "m" (*mptr) : "memory"); -#elif defined(XT_SPL_SOLARIS_LIB) +#elif defined(XT_ATOMIC_SOLARIS_LIB) atomic_swap_32(mptr, val); #else *mptr = val; + xt_log_atomic_error_and_abort(__FUNC__, __FILE__, __LINE__); #endif } -inline xtWord4 xt_atomic_get4(volatile xtWord4 *mptr) -{ - xtWord4 val; - -#ifdef XT_SPL_WIN32_ASM - __asm MOV ECX, mptr - __asm MOV EDX, DWORD PTR [ECX] - __asm MOV val, EDX -#elif defined(XT_SPL_GNUC_X86) - asm volatile ("movl %1,%0" : "=r" (val) : "m" (*mptr) : "memory"); +inline xtWord4 xt_atomic_tas4(volatile xtWord4 *mptr, xtWord4 val) +{ +#ifdef XT_ATOMIC_WIN32_X86 + __asm MOV ECX, mptr + __asm MOV EDX, val + __asm XCHG EDX, DWORD PTR [ECX] + __asm MOV val, EDX +#elif defined(XT_ATOMIC_GNUC_X86) + val = val; + asm volatile ("xchgl %1,%0" : "=r" (val) : "m" (*mptr), "0" (val) : "memory"); +#elif defined(XT_ATOMIC_SOLARIS_LIB) + val = atomic_swap_32(mptr, val); #else - val = *mptr; + *mptr = val; + xt_log_atomic_error_and_abort(__FUNC__, __FILE__, __LINE__); #endif return val; } +/* + * ----------------------------------------------------------------------- + * DIFFERENT TYPES OF LOCKS + */ + +typedef struct XTSpinLock { + volatile xtWord4 spl_lock; +#ifdef XT_NO_ATOMICS + xt_mutex_type spl_mutex; +#endif +#ifdef DEBUG + struct XTThread *spl_locker; +#endif +#ifdef XT_THREAD_LOCK_INFO + XTThreadLockInfoRec spl_lock_info; + const char *spl_name; +#endif +} XTSpinLockRec, *XTSpinLockPtr; + +#ifdef XT_THREAD_LOCK_INFO +#define xt_spinlock_init_with_autoname(a,b) xt_spinlock_init(a,b,LOCKLIST_ARG_SUFFIX(b)) +void xt_spinlock_init(struct XTThread *self, XTSpinLockPtr sp, const char *name); +#else +#define xt_spinlock_init_with_autoname(a,b) xt_spinlock_init(a,b) +void xt_spinlock_init(struct XTThread *self, XTSpinLockPtr sp); +#endif +void xt_spinlock_free(struct XTThread *self, XTSpinLockPtr sp); +xtBool xt_spinlock_spin(XTSpinLockPtr spl); +#ifdef DEBUG +void xt_spinlock_set_thread(XTSpinLockPtr spl); +#endif + /* Code for test and set is derived from code by Larry Zhou and * Google: http://code.google.com/p/google-perftools */ @@ -278,15 +239,15 @@ inline xtWord4 xt_spinlock_set(XTSpinLoc volatile xtWord4 *lck; lck = &spl->spl_lock; -#ifdef XT_SPL_WIN32_ASM +#ifdef XT_ATOMIC_WIN32_X86 __asm MOV ECX, lck __asm MOV EDX, 1 __asm XCHG EDX, DWORD PTR [ECX] __asm MOV prv, EDX -#elif defined(XT_SPL_GNUC_X86) +#elif defined(XT_ATOMIC_GNUC_X86) prv = 1; asm volatile ("xchgl %1,%0" : "=r" (prv) : "m" (*lck), "0" (prv) : "memory"); -#elif defined(XT_SPL_SOLARIS_LIB) +#elif defined(XT_ATOMIC_SOLARIS_LIB) prv = atomic_swap_32(lck, 1); #else /* The default implementation just uses a mutex, and @@ -312,15 +273,15 @@ inline xtWord4 xt_spinlock_reset(XTSpinL spl->spl_locker = NULL; #endif lck = &spl->spl_lock; -#ifdef XT_SPL_WIN32_ASM +#ifdef XT_ATOMIC_WIN32_X86 __asm MOV ECX, lck __asm MOV EDX, 0 __asm XCHG EDX, DWORD PTR [ECX] __asm MOV prv, EDX -#elif defined(XT_SPL_GNUC_X86) +#elif defined(XT_ATOMIC_GNUC_X86) prv = 0; asm volatile ("xchgl %1,%0" : "=r" (prv) : "m" (*lck), "0" (prv) : "memory"); -#elif defined(XT_SPL_SOLARIS_LIB) +#elif defined(XT_ATOMIC_SOLARIS_LIB) prv = atomic_swap_32(lck, 0); #else *lck = 0; @@ -359,9 +320,48 @@ inline void xt_spinlock_unlock(XTSpinLoc #endif } -void xt_unit_test_read_write_locks(struct XTThread *self); -void xt_unit_test_mutex_locks(struct XTThread *self); -void xt_unit_test_create_threads(struct XTThread *self); +/* Possibilities are 2 = align 4 or 2 = align 8 */ +#define XT_XS_LOCK_SHIFT 2 +#define XT_XS_LOCK_ALIGN (1 << XT_XS_LOCK_SHIFT) + +/* This lock is fast for reads but slow for writes. + * Use this lock in situations where you have 99% reads, + * and then some potentially long writes. + */ +typedef struct XTRWMutex { +#ifdef DEBUG + struct XTThread *xs_lock_thread; + u_int xs_inited; +#endif +#ifdef XT_THREAD_LOCK_INFO + XTThreadLockInfoRec xs_lock_info; + const char *xs_name; +#endif + xt_mutex_type xs_lock; + xt_cond_type xs_cond; + volatile xtWord4 xs_state; + volatile xtThreadID xs_xlocker; + union { +#if XT_XS_LOCK_ALIGN == 4 + volatile xtWord4 *xs_rlock_align; +#else + volatile xtWord8 *xs_rlock_align; +#endif + volatile xtWord1 *xs_rlock; + } x; +} XTRWMutexRec, *XTRWMutexPtr; + +#ifdef XT_THREAD_LOCK_INFO +#define xt_rwmutex_init_with_autoname(a,b) xt_rwmutex_init(a,b,LOCKLIST_ARG_SUFFIX(b)) +void xt_rwmutex_init(struct XTThread *self, XTRWMutexPtr xsl, const char *name); +#else +#define xt_rwmutex_init_with_autoname(a,b) xt_rwmutex_init(a,b) +void xt_rwmutex_init(struct XTThread *self, XTRWMutexPtr xsl); +#endif +void xt_rwmutex_free(struct XTThread *self, XTRWMutexPtr xsl); +xtBool xt_rwmutex_xlock(XTRWMutexPtr xsl, xtThreadID thd_id); +xtBool xt_rwmutex_slock(XTRWMutexPtr xsl, xtThreadID thd_id); +xtBool xt_rwmutex_unlock(XTRWMutexPtr xsl, xtThreadID thd_id); #define XT_FAST_LOCK_MAX_WAIT 100 @@ -410,7 +410,7 @@ inline xtBool xt_fastlock_lock(XTFastLoc #endif } -inline void xt_fastlock_unlock(XTFastLockPtr fal, struct XTThread *thread __attribute__((unused))) +inline void xt_fastlock_unlock(XTFastLockPtr fal, struct XTThread *XT_UNUSED(thread)) { if (fal->fal_wait_count) xt_fastlock_wakeup(fal); @@ -423,73 +423,61 @@ inline void xt_fastlock_unlock(XTFastLoc #endif } -typedef struct XTSpinRWLock { - XTSpinLockRec srw_lock; - volatile xtThreadID srw_xlocker; - XTSpinLockRec srw_state_lock; - volatile u_int srw_state; - union { -#if XT_XS_LOCK_ALIGN == 4 - volatile xtWord4 *srw_rlock_align; -#else - volatile xtWord8 *srw_rlock_align; -#endif - volatile xtWord1 *srw_rlock; - } x; +#define XT_SXS_SLOCK_COUNT 2 +typedef struct XTSpinXSLock { + volatile xtWord2 sxs_xlocked; + volatile xtWord2 sxs_rlock_count; + volatile xtWord2 sxs_wait_count; /* The number of readers waiting for the xlocker. */ +#ifdef DEBUG + xtThreadID sxs_locker; +#endif #ifdef XT_THREAD_LOCK_INFO - XTThreadLockInfoRec srw_lock_info; - const char *srw_name; + XTThreadLockInfoRec sxs_lock_info; + const char *sxs_name; #endif - -} XTSpinRWLockRec, *XTSpinRWLockPtr; +} XTSpinXSLockRec, *XTSpinXSLockPtr; #ifdef XT_THREAD_LOCK_INFO -#define xt_spinrwlock_init_with_autoname(a,b) xt_spinrwlock_init(a,b,LOCKLIST_ARG_SUFFIX(b)) -void xt_spinrwlock_init(struct XTThread *self, XTSpinRWLockPtr xsl, const char *name); -#else -#define xt_spinrwlock_init_with_autoname(a,b) xt_spinrwlock_init(a,b) -void xt_spinrwlock_init(struct XTThread *self, XTSpinRWLockPtr xsl); -#endif -void xt_spinrwlock_free(struct XTThread *self, XTSpinRWLockPtr xsl); -xtBool xt_spinrwlock_xlock(XTSpinRWLockPtr xsl, xtThreadID thd_id); -xtBool xt_spinrwlock_slock(XTSpinRWLockPtr xsl, xtThreadID thd_id); -xtBool xt_spinrwlock_unlock(XTSpinRWLockPtr xsl, xtThreadID thd_id); - -typedef struct XTFastRWLock { - XTFastLockRec frw_lock; - struct XTThread *frw_xlocker; - XTSpinLockRec frw_state_lock; - volatile u_int frw_state; - u_int frw_read_waiters; - union { -#if XT_XS_LOCK_ALIGN == 4 - volatile xtWord4 *frw_rlock_align; +#define xt_spinxslock_init_with_autoname(a,b) xt_spinxslock_init(a,b,LOCKLIST_ARG_SUFFIX(b)) +void xt_spinxslock_init(struct XTThread *self, XTSpinXSLockPtr sxs, const char *name); #else - volatile xtWord8 *frw_rlock_align; +#define xt_spinxslock_init_with_autoname(a,b) xt_spinxslock_init(a,b) +void xt_spinxslock_init(struct XTThread *self, XTSpinXSLockPtr sxs); #endif - volatile xtWord1 *frw_rlock; - } x; +void xt_spinxslock_free(struct XTThread *self, XTSpinXSLockPtr sxs); +xtBool xt_spinxslock_xlock(XTSpinXSLockPtr sxs, xtThreadID thd_id); +xtBool xt_spinxslock_slock(XTSpinXSLockPtr sxs); +xtBool xt_spinxslock_unlock(XTSpinXSLockPtr sxs, xtBool xlocked); +typedef struct XTXSMutexLock { + xt_mutex_type xsm_lock; + xt_cond_type xsm_cond; + xt_cond_type xsm_cond_2; + volatile xtThreadID xsm_xlocker; + volatile xtWord2 xsm_rlock_count; + volatile xtWord2 xsm_wait_count; /* The number of readers waiting for the xlocker. */ +#ifdef DEBUG + xtThreadID xsm_locker; +#endif #ifdef XT_THREAD_LOCK_INFO - XTThreadLockInfoRec frw_lock_info; - const char *frw_name; + XTThreadLockInfoRec xsm_lock_info; + const char *xsm_name; #endif - -} XTFastRWLockRec, *XTFastRWLockPtr; +} XTXSMutexRec, *XTXSMutexLockPtr; #ifdef XT_THREAD_LOCK_INFO -#define xt_fastrwlock_init_with_autoname(a,b) xt_fastrwlock_init(a,b,LOCKLIST_ARG_SUFFIX(b)) -void xt_fastrwlock_init(struct XTThread *self, XTFastRWLockPtr frw, const char *name); +#define xt_xsmutex_init_with_autoname(a,b) xt_xsmutex_init(a,b,LOCKLIST_ARG_SUFFIX(b)) +void xt_xsmutex_init(struct XTThread *self, XTXSMutexLockPtr xsm, const char *name); #else -#define xt_fastrwlock_init_with_autoname(a,b) xt_fastrwlock_init(a,b) -void xt_fastrwlock_init(struct XTThread *self, XTFastRWLockPtr frw); +#define xt_xsmutex_init_with_autoname(a,b) xt_xsmutex_init(a,b) +void xt_xsmutex_init(struct XTThread *self, XTXSMutexLockPtr xsm); #endif -void xt_fastrwlock_free(struct XTThread *self, XTFastRWLockPtr frw); -xtBool xt_fastrwlock_xlock(XTFastRWLockPtr frw, struct XTThread *thread); -xtBool xt_fastrwlock_slock(XTFastRWLockPtr frw, struct XTThread *thread); -xtBool xt_fastrwlock_unlock(XTFastRWLockPtr frw, struct XTThread *thread); +void xt_xsmutex_free(struct XTThread *self, XTXSMutexLockPtr xsm); +xtBool xt_xsmutex_xlock(XTXSMutexLockPtr xsm, xtThreadID thd_id); +xtBool xt_xsmutex_slock(XTXSMutexLockPtr xsm, xtThreadID thd_id); +xtBool xt_xsmutex_unlock(XTXSMutexLockPtr xsm, xtThreadID thd_id); typedef struct XTAtomicRWLock { volatile xtWord2 arw_reader_count; @@ -516,6 +504,35 @@ xtBool xt_atomicrwlock_xlock(XTAtomicRWL xtBool xt_atomicrwlock_slock(XTAtomicRWLockPtr xsl); xtBool xt_atomicrwlock_unlock(XTAtomicRWLockPtr xsl, xtBool xlocked); +typedef struct XTSkewRWLock { + volatile xtWord2 srw_reader_count; + volatile xtWord2 srw_xlock_set; + +#ifdef XT_THREAD_LOCK_INFO + XTThreadLockInfoRec srw_lock_info; + const char *srw_name; +#endif +#ifdef DEBUG + xtThreadID srw_locker; +#endif +} XTSkewRWLockRec, *XTSkewRWLockPtr; + +#ifdef XT_THREAD_LOCK_INFO +#define xt_skewrwlock_init_with_autoname(a,b) xt_skewrwlock_init(a,b,LOCKLIST_ARG_SUFFIX(b)) +void xt_skewrwlock_init(struct XTThread *self, XTSkewRWLockPtr xsl, const char *name); +#else +#define xt_skewrwlock_init_with_autoname(a,b) xt_skewrwlock_init(a,b) +void xt_skewrwlock_init(struct XTThread *self, XTSkewRWLockPtr xsl); +#endif +void xt_skewrwlock_free(struct XTThread *self, XTSkewRWLockPtr xsl); +xtBool xt_skewrwlock_xlock(XTSkewRWLockPtr xsl, xtThreadID thr_id); +xtBool xt_skewrwlock_slock(XTSkewRWLockPtr xsl); +xtBool xt_skewrwlock_unlock(XTSkewRWLockPtr xsl, xtBool xlocked); + +void xt_unit_test_read_write_locks(struct XTThread *self); +void xt_unit_test_mutex_locks(struct XTThread *self); +void xt_unit_test_create_threads(struct XTThread *self); + /* * ----------------------------------------------------------------------- * ROW LOCKS === modified file 'storage/pbxt/src/locklist_xt.cc' --- a/storage/pbxt/src/locklist_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/locklist_xt.cc 2009-08-17 11:12:36 +0000 @@ -59,13 +59,13 @@ void xt_thread_lock_info_init(XTThreadLo ptr->li_lock_type = XTThreadLockInfo::RW_LOCK; } -void xt_thread_lock_info_init(XTThreadLockInfoPtr ptr, XTFastRWLock *lock) +void xt_thread_lock_info_init(XTThreadLockInfoPtr ptr, XTXSMutexLock *lock) { ptr->li_fast_rwlock = lock; ptr->li_lock_type = XTThreadLockInfo::FAST_RW_LOCK; } -void xt_thread_lock_info_init(XTThreadLockInfoPtr ptr, XTSpinRWLock *lock) +void xt_thread_lock_info_init(XTThreadLockInfoPtr ptr, XTSpinXSLock *lock) { ptr->li_spin_rwlock = lock; ptr->li_lock_type = XTThreadLockInfo::SPIN_RW_LOCK; @@ -77,6 +77,12 @@ void xt_thread_lock_info_init(XTThreadLo ptr->li_lock_type = XTThreadLockInfo::ATOMIC_RW_LOCK; } +void xt_thread_lock_info_init(XTThreadLockInfoPtr ptr, XTSkewRWLock *lock) +{ + ptr->li_skew_rwlock = lock; + ptr->li_lock_type = XTThreadLockInfo::SKEW_RW_LOCK; +} + void xt_thread_lock_info_free(XTThreadLockInfoPtr ptr) { /* TODO: check to see if it's present in a thread's list */ @@ -163,12 +169,12 @@ void xt_trace_thread_locks(XTThread *sel lock_name = li->li_fast_lock->fal_name; break; case XTThreadLockInfo::FAST_RW_LOCK: - lock_type = "XTFastRWLock"; - lock_name = li->li_fast_rwlock->frw_name; + lock_type = "XTXSMutexLock"; + lock_name = li->li_fast_rwlock->xsm_name; break; case XTThreadLockInfo::SPIN_RW_LOCK: lock_type = "XTSpinRWLock"; - lock_name = li->li_spin_rwlock->srw_name; + lock_name = li->li_spin_rwlock->sxs_name; break; case XTThreadLockInfo::ATOMIC_RW_LOCK: lock_type = "XTAtomicRWLock"; === modified file 'storage/pbxt/src/locklist_xt.h' --- a/storage/pbxt/src/locklist_xt.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/locklist_xt.h 2009-08-17 11:12:36 +0000 @@ -25,7 +25,7 @@ #define __xt_locklist_h__ #ifdef DEBUG -//#define XT_THREAD_LOCK_INFO +#define XT_THREAD_LOCK_INFO #ifndef XT_WIN /* We need DEBUG_LOCKING in order to enable pthread function wrappers */ #define DEBUG_LOCKING @@ -40,9 +40,10 @@ struct XTRWMutex; struct xt_mutex_struct; struct xt_rwlock_struct; struct XTFastLock; -struct XTFastRWLock; -struct XTSpinRWLock; +struct XTXSMutexLock; +struct XTSpinXSLock; struct XTAtomicRWLock; +struct XTSkewRWLock; #ifdef XT_THREAD_LOCK_INFO @@ -61,7 +62,7 @@ struct XTAtomicRWLock; */ typedef struct XTThreadLockInfo { - enum LockType { SPIN_LOCK, RW_MUTEX, MUTEX, RW_LOCK, FAST_LOCK, FAST_RW_LOCK, SPIN_RW_LOCK, ATOMIC_RW_LOCK }; + enum LockType { SPIN_LOCK, RW_MUTEX, MUTEX, RW_LOCK, FAST_LOCK, FAST_RW_LOCK, SPIN_RW_LOCK, ATOMIC_RW_LOCK, SKEW_RW_LOCK }; LockType li_lock_type; @@ -69,11 +70,12 @@ typedef struct XTThreadLockInfo { XTSpinLock *li_spin_lock; // SPIN_LOCK XTRWMutex *li_rw_mutex; // RW_MUTEX XTFastLock *li_fast_lock; // FAST_LOCK - XTFastRWLock *li_fast_rwlock; // FAST_RW_LOCK - XTSpinRWLock *li_spin_rwlock; // SPIN_RW_LOCK + XTXSMutexLock *li_fast_rwlock; // FAST_RW_LOCK + XTSpinXSLock *li_spin_rwlock; // SPIN_RW_LOCK XTAtomicRWLock *li_atomic_rwlock; // ATOMIC_RW_LOCK xt_mutex_struct *li_mutex; // MUTEX xt_rwlock_struct *li_rwlock; // RW_LOCK + XTSkewRWLock *li_skew_rwlock; // SKEW_RW_LOCK }; } XTThreadLockInfoRec, *XTThreadLockInfoPtr; @@ -81,11 +83,12 @@ XTThreadLockInfoRec, *XTThreadLockInfoPt void xt_thread_lock_info_init(XTThreadLockInfoPtr ptr, XTSpinLock *lock); void xt_thread_lock_info_init(XTThreadLockInfoPtr ptr, XTRWMutex *lock); void xt_thread_lock_info_init(XTThreadLockInfoPtr ptr, XTFastLock *lock); -void xt_thread_lock_info_init(XTThreadLockInfoPtr ptr, XTFastRWLock *lock); -void xt_thread_lock_info_init(XTThreadLockInfoPtr ptr, XTSpinRWLock *lock); +void xt_thread_lock_info_init(XTThreadLockInfoPtr ptr, XTXSMutexLock *lock); +void xt_thread_lock_info_init(XTThreadLockInfoPtr ptr, XTSpinXSLock *lock); void xt_thread_lock_info_init(XTThreadLockInfoPtr ptr, XTAtomicRWLock *lock); void xt_thread_lock_info_init(XTThreadLockInfoPtr ptr, xt_mutex_struct *lock); void xt_thread_lock_info_init(XTThreadLockInfoPtr ptr, xt_rwlock_struct *lock); +void xt_thread_lock_info_init(XTThreadLockInfoPtr ptr, XTSkewRWLock *lock); void xt_thread_lock_info_free(XTThreadLockInfoPtr ptr); void xt_thread_lock_info_add_owner (XTThreadLockInfoPtr ptr); === modified file 'storage/pbxt/src/memory_xt.cc' --- a/storage/pbxt/src/memory_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/memory_xt.cc 2009-08-17 11:12:36 +0000 @@ -117,7 +117,7 @@ xtPublic xtBool xt_realloc(XTThreadPtr s return OK; } -xtPublic void xt_free(XTThreadPtr self __attribute__((unused)), void *ptr) +xtPublic void xt_free(XTThreadPtr XT_UNUSED(self), void *ptr) { free(ptr); } @@ -186,7 +186,7 @@ xtPublic void xt_free_ns(void *ptr) free(ptr); } -#ifdef DEBUG +#ifdef DEBUG_MEMORY /* * ----------------------------------------------------------------------- @@ -678,7 +678,7 @@ void xt_mm_memset(void *block, void *des memset(dest, value, size); } -void *xt_mm_malloc(XTThreadPtr self, size_t size, u_int line __attribute__((unused)), c_char *file __attribute__((unused))) +void *xt_mm_malloc(XTThreadPtr self, size_t size, u_int line, c_char *file) { unsigned char *p; @@ -695,6 +695,8 @@ void *xt_mm_malloc(XTThreadPtr self, siz *(p + size + MEM_DEBUG_HDR_SIZE) = MEM_TRAILER_BYTE; *(p + size + MEM_DEBUG_HDR_SIZE + 1L) = MEM_TRAILER_BYTE; + (void) line; + (void) file; #ifdef RECORD_MM xt_lock_mutex(self, &mm_mutex); mm_add_core_ptr(self, p + MEM_DEBUG_HDR_SIZE, 0, line, file); @@ -704,7 +706,7 @@ void *xt_mm_malloc(XTThreadPtr self, siz return p + MEM_DEBUG_HDR_SIZE; } -void *xt_mm_calloc(XTThreadPtr self, size_t size, u_int line __attribute__((unused)), c_char *file __attribute__((unused))) +void *xt_mm_calloc(XTThreadPtr self, size_t size, u_int line, c_char *file) { unsigned char *p; @@ -719,6 +721,8 @@ void *xt_mm_calloc(XTThreadPtr self, siz *(p + size + MEM_DEBUG_HDR_SIZE) = MEM_TRAILER_BYTE; *(p + size + MEM_DEBUG_HDR_SIZE + 1L) = MEM_TRAILER_BYTE; + (void) line; + (void) file; #ifdef RECORD_MM xt_lock_mutex(self, &mm_mutex); mm_add_core_ptr(self, p + MEM_DEBUG_HDR_SIZE, 0, line, file); @@ -849,7 +853,7 @@ void xt_mm_check_ptr(XTThreadPtr self, v xtPublic xtBool xt_init_memory(void) { -#ifdef DEBUG +#ifdef DEBUG_MEMORY XTThreadPtr self = NULL; if (!xt_init_mutex_with_autoname(NULL, &mm_mutex)) @@ -875,7 +879,7 @@ xtPublic void debug_ik_sum(void); xtPublic void xt_exit_memory(void) { -#ifdef DEBUG +#ifdef DEBUG_MEMORY long mm; int i; @@ -919,7 +923,7 @@ xtPublic void xt_exit_memory(void) * MEMORY ALLOCATION UTILITIES */ -#ifdef DEBUG +#ifdef DEBUG_MEMORY char *xt_mm_dup_string(XTThreadPtr self, c_char *str, u_int line, c_char *file) #else char *xt_dup_string(XTThreadPtr self, c_char *str) @@ -931,7 +935,7 @@ char *xt_dup_string(XTThreadPtr self, c_ if (!str) return NULL; len = strlen(str); -#ifdef DEBUG +#ifdef DEBUG_MEMORY new_str = (char *) xt_mm_malloc(self, len + 1, line, file); #else new_str = (char *) xt_malloc(self, len + 1); @@ -1020,7 +1024,7 @@ xtPublic xtBool xt_realloc(XTThreadPtr s return *ptr != NULL; } -xtPublic void xt_free(XTThreadPtr self __attribute__((unused)), void *ptr) +xtPublic void xt_free(XTThreadPtr XT_UNUSED(self), void *ptr) { char *old_ptr; xtWord4 size; === modified file 'storage/pbxt/src/memory_xt.h' --- a/storage/pbxt/src/memory_xt.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/memory_xt.h 2009-08-17 11:12:36 +0000 @@ -30,6 +30,10 @@ struct XTThread; #ifdef DEBUG +#define DEBUG_MEMORY +#endif + +#ifdef DEBUG_MEMORY #define XT_MM_STACK_TRACE 200 #define XT_MM_TRACE_DEPTH 4 @@ -109,7 +113,7 @@ void xt_free_ns(void *ptr); #endif -#ifdef DEBUG +#ifdef DEBUG_MEMORY #define xt_dup_string(t, s) xt_mm_dup_string(t, s, __LINE__, __FILE__) char *xt_mm_dup_string(struct XTThread *self, const char *path, u_int line, const char *file); === modified file 'storage/pbxt/src/myxt_xt.cc' --- a/storage/pbxt/src/myxt_xt.cc 2009-04-02 10:03:14 +0000 +++ b/storage/pbxt/src/myxt_xt.cc 2009-08-18 07:46:53 +0000 @@ -52,12 +52,12 @@ extern pthread_key_t THR_Session; #include "myxt_xt.h" #include "strutil_xt.h" #include "database_xt.h" -#ifdef XT_STREAMING -#include "streaming_xt.h" -#endif #include "cache_xt.h" #include "datalog_xt.h" +static void myxt_bitmap_init(XTThreadPtr self, MX_BITMAP *map, u_int n_bits); +static void myxt_bitmap_free(XTThreadPtr self, MX_BITMAP *map); + #ifdef DRIZZLED #define swap_variables(TYPE, a, b) \ do { \ @@ -143,7 +143,7 @@ static void my_store_blob_length(byte *p static int my_compare_text(MX_CONST_CHARSET_INFO *charset_info, uchar *a, uint a_length, uchar *b, uint b_length, my_bool part_key, - my_bool skip_end_space __attribute__((unused))) + my_bool XT_UNUSED(skip_end_space)) { if (!part_key) /* The last parameter is diff_if_only_endspace_difference, which means @@ -632,7 +632,6 @@ static char *mx_get_length_and_data(Fiel case DRIZZLE_TYPE_DATE: case DRIZZLE_TYPE_NEWDECIMAL: case DRIZZLE_TYPE_ENUM: - case DRIZZLE_TYPE_VIRTUAL: #endif break; } @@ -751,7 +750,6 @@ static void mx_set_length_and_data(Field case DRIZZLE_TYPE_DATE: case DRIZZLE_TYPE_NEWDECIMAL: case DRIZZLE_TYPE_ENUM: - case DRIZZLE_TYPE_VIRTUAL: #endif break; } @@ -764,7 +762,7 @@ static void mx_set_length_and_data(Field bzero(from, field->pack_length()); } -xtPublic void myxt_set_null_row_from_key(XTOpenTablePtr ot __attribute__((unused)), XTIndexPtr ind, xtWord1 *record) +xtPublic void myxt_set_null_row_from_key(XTOpenTablePtr XT_UNUSED(ot), XTIndexPtr ind, xtWord1 *record) { register XTIndexSegRec *keyseg = ind->mi_seg; @@ -800,7 +798,7 @@ xtPublic void myxt_set_default_row_from_ } /* Derived from _mi_put_key_in_record */ -xtPublic xtBool myxt_create_row_from_key(XTOpenTablePtr ot __attribute__((unused)), XTIndexPtr ind, xtWord1 *b_value, u_int key_len, xtWord1 *dest_buff) +xtPublic xtBool myxt_create_row_from_key(XTOpenTablePtr XT_UNUSED(ot), XTIndexPtr ind, xtWord1 *b_value, u_int key_len, xtWord1 *dest_buff) { byte *record = (byte *) dest_buff; register byte *key; @@ -935,8 +933,8 @@ xtPublic xtBool myxt_create_row_from_key #ifdef CHECK_KEYS err: -#endif return FAILED; /* Crashed row */ +#endif } /* @@ -1715,7 +1713,7 @@ xtPublic void myxt_get_column_as_string( /* Required by store() - or an assertion will fail: */ if (table->read_set) - bitmap_set_bit(table->read_set, col_idx); + MX_BIT_SET(table->read_set, col_idx); save = field->ptr; xt_lock_mutex(self, &tab->tab_dic_field_lock); @@ -1743,7 +1741,7 @@ xtPublic xtBool myxt_set_column(XTOpenTa /* Required by store() - or an assertion will fail: */ if (table->write_set) - bitmap_set_bit(table->write_set, col_idx); + MX_BIT_SET(table->write_set, col_idx); mx_set_notnull_in_record(field, buffer); @@ -1875,7 +1873,12 @@ xtPublic void myxt_print_key(XTIndexPtr static void my_close_table(TABLE *table) { -#ifndef DRIZZLED +#ifdef DRIZZLED + TABLE_SHARE *share; + + share = (TABLE_SHARE *) ((char *) table + sizeof(TABLE)); + share->free_table_share(); +#else closefrm(table, 1); // TODO: Q, why did Stewart remove this? #endif xt_free_ns(table); @@ -1885,7 +1888,7 @@ static void my_close_table(TABLE *table) * This function returns NULL if the table cannot be opened * because this is not a MySQL thread. */ -static TABLE *my_open_table(XTThreadPtr self, XTDatabaseHPtr db __attribute__((unused)), XTPathStrPtr tab_path) +static TABLE *my_open_table(XTThreadPtr self, XTDatabaseHPtr XT_UNUSED(db), XTPathStrPtr tab_path) { THD *thd = current_thd; char path_buffer[PATH_MAX]; @@ -1946,6 +1949,18 @@ static TABLE *my_open_table(XTThreadPtr new_lex.current_select= NULL; lex_start(thd); +#ifdef DRIZZLED + share->init(db_name, 0, name, path); + if ((error = open_table_def(thd, share)) || + (error = open_table_from_share(thd, share, "", 0, (uint32_t) READ_ALL, 0, table, OTM_OPEN))) + { + xt_free(self, table); + lex_end(&new_lex); + thd->lex = old_lex; + xt_throw_ulxterr(XT_CONTEXT, XT_ERR_LOADING_MYSQL_DIC, (u_long) error); + return NULL; + } +#else #if MYSQL_VERSION_ID < 60000 #if MYSQL_VERSION_ID < 50123 init_tmp_table_share(share, db_name, 0, name, path); @@ -1980,6 +1995,7 @@ static TABLE *my_open_table(XTThreadPtr xt_throw_ulxterr(XT_CONTEXT, XT_ERR_LOADING_MYSQL_DIC, (u_long) error); return NULL; } +#endif lex_end(&new_lex); thd->lex = old_lex; @@ -1989,8 +2005,10 @@ static TABLE *my_open_table(XTThreadPtr * plugin_shutdown() and reap_plugins() in sql_plugin.cc * from doing their job on shutdown! */ +#ifndef DRIZZLED plugin_unlock(NULL, table->s->db_plugin); table->s->db_plugin = NULL; +#endif return table; } @@ -2069,6 +2087,11 @@ static xtBool my_is_not_null_int4(XTInde return (seg->type == HA_KEYTYPE_LONG_INT && !(seg->flag & HA_NULL_PART)); } +/* MY_BITMAP definition in Drizzle does not like if + * I use a NULL pointer to calculate the offset!? + */ +#define MX_OFFSETOF(x, y) ((size_t)(&((x *) 8)->y) - 8) + /* Derived from ha_myisam::create and mi_create */ static XTIndexPtr my_create_index(XTThreadPtr self, TABLE *table_arg, u_int idx, KEY *index) { @@ -2084,7 +2107,7 @@ static XTIndexPtr my_create_index(XTThre enter_(); - pushsr_(ind, my_deref_index_data, (XTIndexPtr) xt_calloc(self, offsetof(XTIndexRec, mi_seg) + sizeof(XTIndexSegRec) * index->key_parts)); + pushsr_(ind, my_deref_index_data, (XTIndexPtr) xt_calloc(self, MX_OFFSETOF(XTIndexRec, mi_seg) + sizeof(XTIndexSegRec) * index->key_parts)); XT_INDEX_INIT_LOCK(self, ind); xt_init_mutex_with_autoname(self, &ind->mi_flush_lock); @@ -2235,7 +2258,7 @@ static XTIndexPtr my_create_index(XTThre /* NOTE: do not set if the field is only partially in the index!!! */ if (!partial_field) - bitmap_fast_test_and_set(&ind->mi_col_map, field->field_index); + MX_BIT_FAST_TEST_AND_SET(&ind->mi_col_map, field->field_index); } if (key_length > XT_INDEX_MAX_KEY_SIZE) @@ -2243,6 +2266,7 @@ static XTIndexPtr my_create_index(XTThre /* This is the maximum size of the index on disk: */ ind->mi_key_size = key_length; + ind->mi_max_items = (XT_INDEX_PAGE_SIZE-2) / (key_length+XT_RECORD_REF_SIZE); if (ind->mi_fix_key) { /* Special case for not-NULL 4 byte int value: */ @@ -2281,6 +2305,7 @@ static XTIndexPtr my_create_index(XTThre ind->mi_prev_item = xt_prev_branch_item_var; ind->mi_last_item = xt_last_branch_item_var; } + ind->mi_lazy_delete = ind->mi_fix_key && ind->mi_max_items >= 4; XT_NODE_ID(ind->mi_root) = 0; @@ -2344,6 +2369,10 @@ xtPublic void myxt_setup_dictionary(XTTh KEY_PART_INFO *key_part; KEY_PART_INFO *key_part_end; +#ifndef XT_USE_LAZY_DELETE + dic->dic_no_lazy_delete = TRUE; +#endif + dic->dic_ind_cols_req = 0; for (uint i=0; i<TS(my_tab)->keys; i++) { index = &my_tab->key_info[i]; @@ -2602,7 +2631,7 @@ xtPublic void myxt_setup_dictionary(XTTh dic->dic_mysql_rec_size = TS(my_tab)->reclength; } -static u_int my_get_best_superset(XTThreadPtr self __attribute__((unused)), XTDictionaryPtr dic, XTIndexPtr ind) +static u_int my_get_best_superset(XTThreadPtr XT_UNUSED(self), XTDictionaryPtr dic, XTIndexPtr ind) { XTIndexPtr super_ind; u_int super = 0; @@ -2762,7 +2791,7 @@ static void ha_create_dd_index(XTThreadP } } -static char *my_type_to_string(XTThreadPtr self, Field *field, TABLE *my_tab __attribute__((unused))) +static char *my_type_to_string(XTThreadPtr self, Field *field, TABLE *XT_UNUSED(my_tab)) { char buffer[MAX_FIELD_WIDTH + 400], *ptr; String type((char *) buffer, sizeof(buffer), system_charset_info); @@ -2834,7 +2863,7 @@ xtPublic XTDDTable *myxt_create_table_fr * MySQL CHARACTER UTILITIES */ -xtPublic void myxt_static_convert_identifier(XTThreadPtr self __attribute__((unused)), MX_CHARSET_INFO *cs, char *from, char *to, size_t to_len) +xtPublic void myxt_static_convert_identifier(XTThreadPtr XT_UNUSED(self), MX_CHARSET_INFO *cs, char *from, char *to, size_t to_len) { uint errors; @@ -2877,11 +2906,16 @@ xtPublic char *myxt_convert_table_name(X return to; } -xtPublic void myxt_static_convert_table_name(XTThreadPtr self __attribute__((unused)), char *from, char *to, size_t to_len) +xtPublic void myxt_static_convert_table_name(XTThreadPtr XT_UNUSED(self), char *from, char *to, size_t to_len) { tablename_to_filename(from, to, to_len); } +xtPublic void myxt_static_convert_file_name(char *from, char *to, size_t to_len) +{ + filename_to_tablename(from, to, to_len); +} + xtPublic int myxt_strcasecmp(char * a, char *b) { return my_strcasecmp(&my_charset_utf8_general_ci, a, b); @@ -2913,90 +2947,11 @@ xtPublic MX_CHARSET_INFO *myxt_getcharse return &my_charset_utf8_general_ci; } -#ifdef XT_STREAMING -xtPublic xtBool myxt_use_blobs(XTOpenTablePtr ot, void **ret_pbms_table, xtWord1 *rec_buf) -{ - void *pbms_table; - XTTable *tab = ot->ot_table; - u_int idx = 0; - Field *field; - char *blob_ref; - xtWord4 len; - char in_url[PBMS_BLOB_URL_SIZE]; - char *out_url; - - if (!xt_pbms_open_table(&pbms_table, tab->tab_name->ps_path)) - return FAILED; - - for (idx=0; idx<tab->tab_dic.dic_blob_count; idx++) { - field = tab->tab_dic.dic_blob_cols[idx]; - if ((blob_ref = mx_get_length_and_data(field, (char *) rec_buf, &len)) && len) { - xt_strncpy(PBMS_BLOB_URL_SIZE, in_url, blob_ref, len); - - if (!xt_pbms_use_blob(pbms_table, &out_url, in_url, field->field_index)) { - xt_pbms_close_table(pbms_table); - return FAILED; - } - - if (out_url) { - len = strlen(out_url); - mx_set_length_and_data(field, (char *) rec_buf, len, out_url); - } - } - } - *ret_pbms_table = pbms_table; - return OK; -} - -xtPublic void myxt_unuse_blobs(XTOpenTablePtr ot __attribute__((unused)), void *pbms_table) -{ - xt_pbms_close_table(pbms_table); -} - -xtPublic xtBool myxt_retain_blobs(XTOpenTablePtr ot __attribute__((unused)), void *pbms_table, xtRecordID rec_id) -{ - xtBool ok; - PBMSEngineRefRec eng_ref; - - memset(&eng_ref, 0, sizeof(PBMSEngineRefRec)); - XT_SET_DISK_8(eng_ref.er_data, rec_id); - ok = xt_pbms_retain_blobs(pbms_table, &eng_ref); - xt_pbms_close_table(pbms_table); - return ok; -} - -xtPublic void myxt_release_blobs(XTOpenTablePtr ot, xtWord1 *rec_buf, xtRecordID rec_id) -{ - void *pbms_table; - XTTable *tab = ot->ot_table; - u_int idx = 0; - Field *field; - char *blob_ref; - xtWord4 len; - char in_url[PBMS_BLOB_URL_SIZE]; - PBMSEngineRefRec eng_ref; - - memset(&eng_ref, 0, sizeof(PBMSEngineRefRec)); - XT_SET_DISK_8(eng_ref.er_data, rec_id); - - if (!xt_pbms_open_table(&pbms_table, tab->tab_name->ps_path)) - return; - - for (idx=0; idx<tab->tab_dic.dic_blob_count; idx++) { - field = tab->tab_dic.dic_blob_cols[idx]; - if ((blob_ref = mx_get_length_and_data(field, (char *) rec_buf, &len)) && len) { - xt_strncpy(PBMS_BLOB_URL_SIZE, in_url, blob_ref, len); - - xt_pbms_release_blob(pbms_table, in_url, field->field_index, &eng_ref); - } - } - - xt_pbms_close_table(pbms_table); -} -#endif // XT_STREAMING - xtPublic void *myxt_create_thread() { +#ifdef DRIZZLED + return (void *) 1; +#else THD *new_thd; if (my_thread_init()) { @@ -3015,8 +2970,14 @@ xtPublic void *myxt_create_thread() lex_start(new_thd); return (void *) new_thd; +#endif } +#ifdef DRIZZLED +xtPublic void myxt_destroy_thread(void *, xtBool) +{ +} +#else xtPublic void myxt_destroy_thread(void *thread, xtBool end_threads) { THD *thd = (THD *) thread; @@ -3043,6 +3004,7 @@ xtPublic void myxt_destroy_thread(void * if (end_threads) my_thread_end(); } +#endif xtPublic XTThreadPtr myxt_get_self() { @@ -3182,7 +3144,7 @@ xtPublic void myxt_get_status(XTThreadPt * MySQL Bit Maps */ -xtPublic void myxt_bitmap_init(XTThreadPtr self, MY_BITMAP *map, u_int n_bits) +static void myxt_bitmap_init(XTThreadPtr self, MX_BITMAP *map, u_int n_bits) { my_bitmap_map *buf; uint size_in_bytes = (((n_bits) + 31) / 32) * 4; @@ -3194,7 +3156,7 @@ xtPublic void myxt_bitmap_init(XTThreadP bitmap_clear_all(map); } -xtPublic void myxt_bitmap_free(XTThreadPtr self, MY_BITMAP *map) +static void myxt_bitmap_free(XTThreadPtr self, MX_BITMAP *map) { if (map->bitmap) { xt_free(self, map->bitmap); === modified file 'storage/pbxt/src/myxt_xt.h' --- a/storage/pbxt/src/myxt_xt.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/myxt_xt.h 2009-08-18 07:46:53 +0000 @@ -70,6 +70,7 @@ XTDDTable *myxt_create_table_from_table( void myxt_static_convert_identifier(XTThreadPtr self, struct charset_info_st *cs, char *from, char *to, size_t to_len); char *myxt_convert_identifier(XTThreadPtr self, struct charset_info_st *cs, char *from); void myxt_static_convert_table_name(XTThreadPtr self, char *from, char *to, size_t to_len); +void myxt_static_convert_file_name(char *from, char *to, size_t to_len); char *myxt_convert_table_name(XTThreadPtr self, char *from); int myxt_strcasecmp(char * a, char *b); int myxt_isspace(struct charset_info_st *cs, char a); @@ -78,13 +79,6 @@ int myxt_isdigit(struct charset_info_s struct charset_info_st *myxt_getcharset(bool convert); -#ifdef XT_STREAMING -xtBool myxt_use_blobs(XTOpenTablePtr ot, void **ret_pbms_table, xtWord1 *rec_buf); -void myxt_unuse_blobs(XTOpenTablePtr ot, void *pbms_table); -xtBool myxt_retain_blobs(XTOpenTablePtr ot, void *pbms_table, xtRecordID record); -void myxt_release_blobs(XTOpenTablePtr ot, xtWord1 *rec_buf, xtRecordID record); -#endif - void *myxt_create_thread(); void myxt_destroy_thread(void *thread, xtBool end_threads); XTThreadPtr myxt_get_self(); @@ -92,9 +86,6 @@ XTThreadPtr myxt_get_self(); int myxt_statistics_fill_table(XTThreadPtr self, void *th, void *ta, void *co, MX_CONST void *ch); void myxt_get_status(XTThreadPtr self, XTStringBufferPtr strbuf); -void myxt_bitmap_init(XTThreadPtr self, MY_BITMAP *map, u_int n_bits); -void myxt_bitmap_free(XTThreadPtr self, MY_BITMAP *map); - class XTDDColumnFactory { public: === modified file 'storage/pbxt/src/pbms.h' --- a/storage/pbxt/src/pbms.h 2009-05-09 04:01:53 +0000 +++ b/storage/pbxt/src/pbms.h 2009-08-18 07:46:53 +0000 @@ -16,7 +16,8 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA * - * Paul McCullagh + * Original author: Paul McCullagh + * Continued development: Barry Leslie * H&G2JCtL * * 2007-06-01 @@ -37,21 +38,26 @@ #include <dirent.h> #include <signal.h> #include <ctype.h> +#include <errno.h> + #ifdef USE_PRAGMA_INTERFACE #pragma interface /* gcc class implementation */ #endif +/* 2 10 1 10 20 10 10 20 20 + * Format: "~*"<db_id><'~' || '_'><tab_id>"-"<blob_id>"-"<auth_code>"-"<server_id>"-"<blob_ref_id>"-"<blob_size> + */ +//If URL_FMT changes do not forget to update couldBeURL() in this file. + +#define URL_FMT "~*%lu%c%lu-%llu-%lx-%lu-%llu-%llu" + #define MS_SHARED_MEMORY_MAGIC 0x7E9A120C #define MS_ENGINE_VERSION 1 -#define MS_CALLBACK_VERSION 1 -#define MS_SHARED_MEMORY_VERSION 1 -#define MS_ENGINE_LIST_SIZE 80 +#define MS_CALLBACK_VERSION 4 +#define MS_SHARED_MEMORY_VERSION 2 +#define MS_ENGINE_LIST_SIZE 10 #define MS_TEMP_FILE_PREFIX "pbms_temp_" -#define MS_TEMP_FILE_PREFIX "pbms_temp_" - -#define MS_RESULT_MESSAGE_SIZE 300 -#define MS_RESULT_STACK_SIZE 200 #define MS_BLOB_HANDLE_SIZE 300 @@ -68,146 +74,81 @@ #define MS_ERR_UNKNOWN_DB 8 #define MS_ERR_REMOVING_REPO 9 #define MS_ERR_DATABASE_DELETED 10 +#define MS_ERR_DUPLICATE 11 /* Attempt to insert a duplicate key into a system table. */ +#define MS_ERR_INVALID_RECORD 12 +#define MS_ERR_RECOVERY_IN_PROGRESS 13 +#define MS_ERR_DUPLICATE_DB 14 +#define MS_ERR_DUPLICATE_DB_ID 15 +#define MS_ERR_INVALID_OPERATION 16 #define MS_LOCK_NONE 0 #define MS_LOCK_READONLY 1 #define MS_LOCK_READ_WRITE 2 -#define MS_XACT_NONE 0 -#define MS_XACT_BEGIN 1 -#define MS_XACT_COMMIT 2 -#define MS_XACT_ROLLBACK 3 - -#define PBMS_ENGINE_REF_LEN 8 -#define PBMS_BLOB_URL_SIZE 200 +#define PBMS_BLOB_URL_SIZE 120 #define PBMS_FIELD_COL_SIZE 128 #define PBMS_FIELD_COND_SIZE 300 +#define MS_RESULT_MESSAGE_SIZE 300 +#define MS_RESULT_STACK_SIZE 200 + +typedef struct PBMSResultRec { + int mr_code; /* Engine specific error code. */ + char mr_message[MS_RESULT_MESSAGE_SIZE]; /* Error message, required if non-zero return code. */ + char mr_stack[MS_RESULT_STACK_SIZE]; /* Trace information about where the error occurred. */ +} PBMSResultRec, *PBMSResultPtr; + + typedef struct PBMSBlobID { + u_int32_t bi_db_id; u_int64_t bi_blob_size; u_int64_t bi_blob_id; // or repo file offset if type = REPO + u_int64_t bi_blob_ref_id; u_int32_t bi_tab_id; // or repo ID if type = REPO u_int32_t bi_auth_code; u_int32_t bi_blob_type; } PBMSBlobIDRec, *PBMSBlobIDPtr; -typedef struct PBMSResultRec { - int mr_code; /* Engine specific error code. */ - char mr_message[MS_RESULT_MESSAGE_SIZE]; /* Error message, required if non-zero return code. */ - char mr_stack[MS_RESULT_STACK_SIZE]; /* Trace information about where the error occurred. */ -} PBMSResultRec, *PBMSResultPtr; - -typedef struct PBMSEngineRefRec { - unsigned char er_data[PBMS_ENGINE_REF_LEN]; -} PBMSEngineRefRec, *PBMSEngineRefPtr; - typedef struct PBMSBlobURL { char bu_data[PBMS_BLOB_URL_SIZE]; } PBMSBlobURLRec, *PBMSBlobURLPtr; -typedef struct PBMSFieldRef { - char fr_column[PBMS_FIELD_COL_SIZE]; - char fr_cond[PBMS_FIELD_COND_SIZE]; -} PBMSFieldRefRec, *PBMSFieldRefPtr; -/* - * The engine must free its resources for the given thread. - */ -typedef void (*MSCloseConnFunc)(void *thd); - -/* Before access BLOBs of a table, the streaming engine will open the table. - * Open tables are managed as a pool by the streaming engine. - * When a request is received, the streaming engine will ask all - * registered engine to open the table. The engine must return a NULL - * open_table pointer if it does not handle the table. - * A callback allows an engine to request all open tables to be - * closed by the streaming engine. - */ -typedef int (*MSOpenTableFunc)(void *thd, const char *table_url, void **open_table, PBMSResultPtr result); -typedef void (*MSCloseTableFunc)(void *thd, void *open_table); - -/* - * When the streaming engine wants to use an open table handle from the - * pool, it calls the lock table function. - */ -typedef int (*MSLockTableFunc)(void *thd, int *xact, void *open_table, int lock_type, PBMSResultPtr result); -typedef int (*MSUnlockTableFunc)(void *thd, int xact, void *open_table, PBMSResultPtr result); - -/* This function is used to locate and send a BLOB on the given stream. - */ -typedef int (*MSSendBLOBFunc)(void *thd, void *open_table, const char *blob_column, const char *blob_url, void *stream, PBMSResultPtr result); - -/* - * Lookup and engine reference, and return readable text. - */ -typedef int (*MSLookupRefFunc)(void *thd, void *open_table, unsigned short col_index, PBMSEngineRefPtr eng_ref, PBMSFieldRefPtr feild_ref, PBMSResultPtr result); - typedef struct PBMSEngineRec { int ms_version; /* MS_ENGINE_VERSION */ int ms_index; /* The index into the engine list. */ int ms_removing; /* TRUE (1) if the engine is being removed. */ - const char *ms_engine_name; - void *ms_engine_info; - MSCloseConnFunc ms_close_conn; - MSOpenTableFunc ms_open_table; - MSCloseTableFunc ms_close_table; - MSLockTableFunc ms_lock_table; - MSUnlockTableFunc ms_unlock_table; - MSSendBLOBFunc ms_send_blob; - MSLookupRefFunc ms_lookup_ref; + int ms_internal; /* TRUE (1) if the engine is supported directly in the mysq/drizzle handler code . */ + char ms_engine_name[32]; } PBMSEngineRec, *PBMSEnginePtr; /* * This function should never be called directly, it is called * by deregisterEngine() below. */ -typedef void (*ECDeregisterdFunc)(PBMSEnginePtr engine); - -typedef void (*ECTableCloseAllFunc)(const char *table_url); - -typedef int (*ECSetContentLenFunc)(void *stream, off_t len, PBMSResultPtr result); - -typedef int (*ECWriteHeadFunc)(void *stream, PBMSResultPtr result); - -typedef int (*ECWriteStreamFunc)(void *stream, void *buffer, size_t len, PBMSResultPtr result); - -/* - * The engine should call this function from - * its own close connection function! - */ -typedef int (*ECCloseConnFunc)(void *thd, PBMSResultPtr result); +typedef void (*ECRegisterdFunc)(PBMSEnginePtr engine); -/* - * Call this function before retaining or releasing BLOBs in a row. - */ -typedef int (*ECOpenTableFunc)(void **open_table, char *table_path, PBMSResultPtr result); +typedef void (*ECDeregisterdFunc)(PBMSEnginePtr engine); /* - * Call this function when the operation is complete. + * Call this function to store a BLOB in the repository the BLOB's + * URL will be returned. The returned URL buffer is expected to be atleast + * PBMS_BLOB_URL_SIZE long. + * + * The BLOB URL must still be retained or it will automaticly be deleted after a timeout expires. */ -typedef void (*ECCloseTableFunc)(void *open_table); +typedef int (*ECCreateBlobsFunc)(bool built_in, const char *db_name, const char *tab_name, char *blob, size_t blob_len, char *blob_url, unsigned short col_index, PBMSResultPtr result); /* * Call this function for each BLOB to be retained. When a BLOB is used, the - * URL may be changed. The returned URL is valid as long as the the - * table is open. + * URL may be changed. The returned URL buffer is expected to be atleast + * PBMS_BLOB_URL_SIZE long. * * The returned URL must be inserted into the row in place of the given * URL. */ -typedef int (*ECUseBlobFunc)(void *open_table, char **ret_blob_url, char *blob_url, unsigned short col_index, PBMSResultPtr result); - -/* - * Reference Blobs that has been uploaded to the streaming engine. - * - * All BLOBs specified by the use blob function are retained by - * this function. - * - * The engine reference is a (unaligned) 8 byte value which - * identifies the row that the BLOBs are in. - */ -typedef int (*ECRetainBlobsFunc)(void *open_table, PBMSEngineRefPtr eng_ref, PBMSResultPtr result); +typedef int (*ECRetainBlobsFunc)(bool built_in, const char *db_name, const char *tab_name, char *ret_blob_url, char *blob_url, unsigned short col_index, PBMSResultPtr result); /* * If a row containing a BLOB is deleted, then the BLOBs in the @@ -216,27 +157,24 @@ typedef int (*ECRetainBlobsFunc)(void *o * Note: if a table is dropped, all the BLOBs referenced by the * table are automatically released. */ -typedef int (*ECReleaseBlobFunc)(void *open_table, char *blob_url, unsigned short col_index, PBMSEngineRefPtr eng_ref, PBMSResultPtr result); +typedef int (*ECReleaseBlobFunc)(bool built_in, const char *db_name, const char *tab_name, char *blob_url, PBMSResultPtr result); -typedef int (*ECDropTable)(const char *table_path, PBMSResultPtr result); +typedef int (*ECDropTable)(bool built_in, const char *db_name, const char *tab_name, PBMSResultPtr result); -typedef int (*ECRenameTable)(const char *from_table, const char *to_table, PBMSResultPtr result); +typedef int (*ECRenameTable)(bool built_in, const char *db_name, const char *from_table, const char *to_table, PBMSResultPtr result); + +typedef void (*ECCallCompleted)(bool built_in, bool ok); typedef struct PBMSCallbacksRec { int cb_version; /* MS_CALLBACK_VERSION */ + ECRegisterdFunc cb_register; ECDeregisterdFunc cb_deregister; - ECTableCloseAllFunc cb_table_close_all; - ECSetContentLenFunc cb_set_cont_len; - ECWriteHeadFunc cb_write_head; - ECWriteStreamFunc cb_write_stream; - ECCloseConnFunc cb_close_conn; - ECOpenTableFunc cb_open_table; - ECCloseTableFunc cb_close_table; - ECUseBlobFunc cb_use_blob; - ECRetainBlobsFunc cb_retain_blobs; + ECCreateBlobsFunc cb_create_blob; + ECRetainBlobsFunc cb_retain_blob; ECReleaseBlobFunc cb_release_blob; ECDropTable cb_drop_table; ECRenameTable cb_rename_table; + ECCallCompleted cb_completed; } PBMSCallbacksRec, *PBMSCallbacksPtr; typedef struct PBMSSharedMemoryRec { @@ -251,24 +189,18 @@ typedef struct PBMSSharedMemoryRec { PBMSEnginePtr sm_engine_list[MS_ENGINE_LIST_SIZE]; } PBMSSharedMemoryRec, *PBMSSharedMemoryPtr; -#ifndef PBMS_API -#ifndef PBMS_CLIENT_API -Please define he value of PBMS_API -#endif -#else +#ifdef PBMS_API class PBMS_API { private: const char *temp_prefix[3]; + bool built_in; public: PBMS_API(): sharedMemory(NULL) { int i = 0; temp_prefix[i++] = MS_TEMP_FILE_PREFIX; -#ifdef MS_TEMP_FILE_PREFIX - temp_prefix[i++] = MS_TEMP_FILE_PREFIX; -#endif temp_prefix[i++] = NULL; } @@ -276,6 +208,43 @@ public: ~PBMS_API() { } /* + * This method is called by the PBMS engine during startup. + */ + int PBMSStartup(PBMSCallbacksPtr callbacks, PBMSResultPtr result) { + int err; + + deleteTempFiles(); + err = getSharedMemory(true, result); + if (!err) + sharedMemory->sm_callbacks = callbacks; + + return err; + } + + /* + * This method is called by the PBMS engine during startup. + */ + void PBMSShutdown() { + + if (!sharedMemory) + return; + + lock(); + sharedMemory->sm_callbacks = NULL; + + bool empty = true; + for (int i=0; i<sharedMemory->sm_list_len && empty; i++) { + if (sharedMemory->sm_engine_list[i]) + empty = false; + } + + unlock(); + + if (empty) + removeSharedMemory(); + } + + /* * Register the engine with the Stream Engine. */ int registerEngine(PBMSEnginePtr engine, PBMSResultPtr result) { @@ -283,6 +252,7 @@ public: deleteTempFiles(); + // The first engine to register creates the shared memory. if ((err = getSharedMemory(true, result))) return err; @@ -292,6 +262,10 @@ public: engine->ms_index = i; if (i >= sharedMemory->sm_list_len) sharedMemory->sm_list_len = i+1; + if (sharedMemory->sm_callbacks) + sharedMemory->sm_callbacks->cb_register(engine); + + built_in = (engine->ms_internal == 1); return MS_OK; } } @@ -322,7 +296,7 @@ public: PBMSResultRec result; int err; - if ((err = getSharedMemory(true, &result))) + if ((err = getSharedMemory(false, &result))) return; lock(); @@ -342,207 +316,98 @@ public: unlock(); - if (empty) { - char temp_file[100]; - - sharedMemory->sm_magic = 0; - free(sharedMemory); - sharedMemory = NULL; - const char **prefix = temp_prefix; - while (*prefix) { - getTempFileName(temp_file, *prefix, getpid()); - unlink(temp_file); - prefix++; - } - } + if (empty) + removeSharedMemory(); } - void closeAllTables(const char *table_url) + void removeSharedMemory() { - PBMSResultRec result; - int err; - - if ((err = getSharedMemory(true, &result))) - return; + const char **prefix = temp_prefix; + char temp_file[100]; + // Do not remove the sharfed memory until after + // the PBMS engine has shutdown. if (sharedMemory->sm_callbacks) - sharedMemory->sm_callbacks->cb_table_close_all(table_url); - } - - int setContentLength(void *stream, off_t len, PBMSResultPtr result) - { - int err; - - if ((err = getSharedMemory(true, result))) - return err; - - return sharedMemory->sm_callbacks->cb_set_cont_len(stream, len, result); - } - - int writeHead(void *stream, PBMSResultPtr result) - { - int err; - - if ((err = getSharedMemory(true, result))) - return err; - - return sharedMemory->sm_callbacks->cb_write_head(stream, result); - } - - int writeStream(void *stream, void *buffer, size_t len, PBMSResultPtr result) - { - int err; - - if ((err = getSharedMemory(true, result))) - return err; - - return sharedMemory->sm_callbacks->cb_write_stream(stream, buffer, len, result); - } - - int closeConn(void *thd, PBMSResultPtr result) - { - int err; - - if ((err = getSharedMemory(true, result))) - return err; - - if (!sharedMemory->sm_callbacks) - return MS_OK; - - return sharedMemory->sm_callbacks->cb_close_conn(thd, result); - } - - int openTable(void **open_table, char *table_path, PBMSResultPtr result) - { - int err; - - if ((err = getSharedMemory(true, result))) - return err; - - if (!sharedMemory->sm_callbacks) { - *open_table = NULL; - return MS_OK; + return; + + sharedMemory->sm_magic = 0; + free(sharedMemory); + sharedMemory = NULL; + + while (*prefix) { + getTempFileName(temp_file, *prefix, getpid()); + unlink(temp_file); + prefix++; } - - return sharedMemory->sm_callbacks->cb_open_table(open_table, table_path, result); - } - - int closeTable(void *open_table, PBMSResultPtr result) - { - int err; - - if ((err = getSharedMemory(true, result))) - return err; - - if (sharedMemory->sm_callbacks && open_table) - sharedMemory->sm_callbacks->cb_close_table(open_table); - return MS_OK; } - - int couldBeURL(char *blob_url) - /* ~*test/~1-150-2b5e0a7-0[*<blob size>][.ext] */ - /* ~*test/_1-150-2b5e0a7-0[*<blob size>][.ext] */ - { - char *ptr; - size_t len; - bool have_blob_size = false; - - if (blob_url) { - if ((len = strlen(blob_url))) { - /* Too short: */ - if (len <= 10) - return 0; - - /* Required prefix: */ - /* NOTE: ~> is deprecated v0.5.4+, now use ~* */ - if (*blob_url != '~' || (*(blob_url + 1) != '>' && *(blob_url + 1) != '*')) - return 0; - - ptr = blob_url + len - 1; - - /* Allow for an optional extension: */ - if (!isdigit(*ptr)) { - while (ptr > blob_url && *ptr != '/' && *ptr != '.') - ptr--; - if (ptr == blob_url || *ptr != '.') - return 0; - if (ptr == blob_url || !isdigit(*ptr)) - return 0; - } - // field 1: server id OR blob size - do_again: - while (ptr > blob_url && isdigit(*ptr)) - ptr--; - - if (ptr != blob_url && *ptr == '*' && !have_blob_size) { - ptr--; - have_blob_size = true; - goto do_again; - } - - if (ptr == blob_url || *ptr != '-') - return 0; - - - // field 2: Authoration code - ptr--; - if (!isxdigit(*ptr)) - return 0; - - while (ptr > blob_url && isxdigit(*ptr)) - ptr--; - - if (ptr == blob_url || *ptr != '-') - return 0; - - // field 3:offset - ptr--; - if (!isxdigit(*ptr)) - return 0; - - while (ptr > blob_url && isdigit(*ptr)) - ptr--; - - if (ptr == blob_url || *ptr != '-') - return 0; - - - // field 4:Table id - ptr--; - if (!isdigit(*ptr)) - return 0; - - while (ptr > blob_url && isdigit(*ptr)) - ptr--; - - /* NOTE: ^ and : are deprecated v0.5.4+, now use ! and ~ */ - if (ptr == blob_url || (*ptr != '^' && *ptr != ':' && *ptr != '_' && *ptr != '~')) - return 0; - ptr--; - - if (ptr == blob_url || *ptr != '/') - return 0; - ptr--; - if (ptr == blob_url) - return 0; - return 1; + int couldBeURL(char *blob_url, int size) + { + if (blob_url && (size < PBMS_BLOB_URL_SIZE)) { + char buffer[PBMS_BLOB_URL_SIZE+1]; + u_int32_t db_id = 0; + u_int32_t tab_id = 0; + u_int64_t blob_id = 0; + u_int64_t blob_ref_id = 0; + u_int64_t blob_size = 0; + u_int32_t auth_code = 0; + u_int32_t server_id = 0; + char type, junk[5]; + int scanned; + + junk[0] = 0; + if (blob_url[size]) { // There is no guarantee that the URL will be null terminated. + memcpy(buffer, blob_url, size); + buffer[size] = 0; + blob_url = buffer; + } + + scanned = sscanf(blob_url, URL_FMT"%4s", &db_id, &type, &tab_id, &blob_id, &auth_code, &server_id, &blob_ref_id, &blob_size, junk); + if (scanned != 8) {// If junk is found at the end this will also result in an invalid URL. + printf("Bad URL \"%s\": scanned = %d, junk: %d, %d, %d, %d\n", blob_url, scanned, junk[0], junk[1], junk[2], junk[3]); + return 0; } + + if (junk[0] || (type != '~' && type != '_')) { + printf("Bad URL \"%s\": scanned = %d, junk: %d, %d, %d, %d\n", blob_url, scanned, junk[0], junk[1], junk[2], junk[3]); + return 0; + } + + return 1; } + return 0; } - - int useBlob(void *open_table, char **ret_blob_url, char *blob_url, unsigned short col_index, PBMSResultPtr result) + + int retainBlob(const char *db_name, const char *tab_name, char *ret_blob_url, char *blob_url, size_t blob_size, unsigned short col_index, PBMSResultPtr result) { int err; + char safe_url[PBMS_BLOB_URL_SIZE+1]; - if ((err = getSharedMemory(true, result))) + + if ((err = getSharedMemory(false, result))) return err; - if (!couldBeURL(blob_url)) { - *ret_blob_url = NULL; - return MS_OK; + if (!couldBeURL(blob_url, blob_size)) { + + if (!sharedMemory->sm_callbacks) { + *ret_blob_url = 0; + return MS_OK; + } + err = sharedMemory->sm_callbacks->cb_create_blob(built_in, db_name, tab_name, blob_url, blob_size, ret_blob_url, col_index, result); + if (err) + return err; + + blob_url = ret_blob_url; + } else { + // Make sure the url is a C string: + if (blob_url[blob_size]) { + memcpy(safe_url, blob_url, blob_size); + safe_url[blob_size] = 0; + blob_url = safe_url; + } } + if (!sharedMemory->sm_callbacks) { result->mr_code = MS_ERR_INCORRECT_URL; @@ -551,64 +416,71 @@ public: return MS_ERR_INCORRECT_URL; } - return sharedMemory->sm_callbacks->cb_use_blob(open_table, ret_blob_url, blob_url, col_index, result); + return sharedMemory->sm_callbacks->cb_retain_blob(built_in, db_name, tab_name, ret_blob_url, blob_url, col_index, result); } - int retainBlobs(void *open_table, PBMSEngineRefPtr eng_ref, PBMSResultPtr result) + int releaseBlob(const char *db_name, const char *tab_name, char *blob_url, size_t blob_size, PBMSResultPtr result) { int err; + char safe_url[PBMS_BLOB_URL_SIZE+1]; - if ((err = getSharedMemory(true, result))) + if ((err = getSharedMemory(false, result))) return err; if (!sharedMemory->sm_callbacks) return MS_OK; - return sharedMemory->sm_callbacks->cb_retain_blobs(open_table, eng_ref, result); + if (!couldBeURL(blob_url, blob_size)) + return MS_OK; + + if (blob_url[blob_size]) { + memcpy(safe_url, blob_url, blob_size); + safe_url[blob_size] = 0; + blob_url = safe_url; + } + + return sharedMemory->sm_callbacks->cb_release_blob(built_in, db_name, tab_name, blob_url, result); } - int releaseBlob(void *open_table, char *blob_url, unsigned short col_index, PBMSEngineRefPtr eng_ref, PBMSResultPtr result) + int dropTable(const char *db_name, const char *tab_name, PBMSResultPtr result) { int err; - if ((err = getSharedMemory(true, result))) + if ((err = getSharedMemory(false, result))) return err; if (!sharedMemory->sm_callbacks) return MS_OK; - - if (!couldBeURL(blob_url)) - return MS_OK; - - return sharedMemory->sm_callbacks->cb_release_blob(open_table, blob_url, col_index, eng_ref, result); + + return sharedMemory->sm_callbacks->cb_drop_table(built_in, db_name, tab_name, result); } - int dropTable(const char *table_path, PBMSResultPtr result) + int renameTable(const char *db_name, const char *from_table, const char *to_table, PBMSResultPtr result) { int err; - if ((err = getSharedMemory(true, result))) + if ((err = getSharedMemory(false, result))) return err; if (!sharedMemory->sm_callbacks) return MS_OK; - return sharedMemory->sm_callbacks->cb_drop_table(table_path, result); + return sharedMemory->sm_callbacks->cb_rename_table(built_in, db_name, from_table, to_table, result); } - int renameTable(const char *from_table, const char *to_table, PBMSResultPtr result) + void completed(int ok) { - int err; + PBMSResultRec result; - if ((err = getSharedMemory(true, result))) - return err; + if (getSharedMemory(false, &result)) + return; if (!sharedMemory->sm_callbacks) - return MS_OK; + return; - return sharedMemory->sm_callbacks->cb_rename_table(from_table, to_table, result); + sharedMemory->sm_callbacks->cb_completed(built_in, ok); } - + volatile PBMSSharedMemoryPtr sharedMemory; private: @@ -618,7 +490,6 @@ private: int r; char temp_file[100]; const char **prefix = temp_prefix; - void *tmp_p = NULL; if (sharedMemory) return MS_OK; @@ -644,8 +515,7 @@ private: } buffer[tfer] = 0; - sscanf(buffer, "%p", &tmp_p); - sharedMemory = (PBMSSharedMemoryPtr) tmp_p; + sscanf(buffer, "%p", &sharedMemory); if (!sharedMemory || sharedMemory->sm_magic != MS_SHARED_MEMORY_MAGIC) { if (!create) return MS_OK; @@ -661,9 +531,9 @@ private: return setOSResult(errno, "fseek", temp_file, result); } - sprintf(buffer, "%p", (void *) sharedMemory); + sprintf(buffer, "%p", sharedMemory); tfer = write(tmp_f, buffer, strlen(buffer)); - if (tfer != (ssize_t) strlen(buffer)) { + if (tfer != strlen(buffer)) { close(tmp_f); return setOSResult(errno, "write", temp_file, result); } @@ -782,19 +652,20 @@ private: void deleteTempFiles() { - struct dirent *entry; + struct dirent *entry; struct dirent *result; DIR *odir; int err; - size_t sz; + size_t sz; char temp_file[100]; -#ifdef XT_SOLARIS +#ifdef __sun sz = sizeof(struct dirent) + pathconf("/tmp/", _PC_NAME_MAX); // Solaris, see readdir(3C) #else sz = sizeof(struct dirent); #endif - entry = (struct dirent*)malloc(sz); + if (!(entry = (struct dirent *) malloc(sz))) + return; if (!(odir = opendir("/tmp/"))) return; err = readdir_r(odir, entry, &result); @@ -846,25 +717,25 @@ extern void PBMSDeinitBlobStreamingThrea extern void PBMSGetError(void *v_bs_thread, PBMSResultPtr result); /* -* PBMSCreateBlob():Creates a new blob in the database of the given size. cont_type can be NULL. +* PBMSCreateBlob():Creates a new blob in the database of the given size. */ -extern bool PBMSCreateBlob(PBMSBlobIDPtr blob_id, char *database_name, char *cont_type, u_int64_t size); +extern bool PBMSCreateBlob(PBMSBlobIDPtr blob_id, char *database_name, u_int64_t size); /* * PBMSWriteBlob():Write the data to the blob in one or more chunks. The total size of all the chuncks of * data written to the blob must match the size specified when the blob was created. */ -extern bool PBMSWriteBlob(PBMSBlobIDPtr blob_id, char *database_name, char *data, size_t size, size_t offset); +extern bool PBMSWriteBlob(PBMSBlobIDPtr blob_id, char *data, size_t size, size_t offset); /* * PBMSReadBlob():Read the blob data out of the blob in one or more chunks. */ -extern bool PBMSReadBlob(PBMSBlobIDPtr blob_id, char *database_name, char *buffer, size_t *size, size_t offset); +extern bool PBMSReadBlob(PBMSBlobIDPtr blob_id, char *buffer, size_t *size, size_t offset); /* * PBMSIDToURL():Convert a blob id to a blob URL. The 'url' buffer must be atleast PBMS_BLOB_URL_SIZE bytes in size. */ -extern bool PBMSIDToURL(PBMSBlobIDPtr blob_id, char *database_name, char *url); +extern bool PBMSIDToURL(PBMSBlobIDPtr blob_id, char *url); /* * PBMSIDToURL():Convert a blob URL to a blob ID. === added file 'storage/pbxt/src/pbms_enabled.cc' --- a/storage/pbxt/src/pbms_enabled.cc 1970-01-01 00:00:00 +0000 +++ b/storage/pbxt/src/pbms_enabled.cc 2009-08-18 07:46:53 +0000 @@ -0,0 +1,238 @@ +/* Copyright (c) 2009 PrimeBase Technologies GmbH, Germany + * + * PrimeBase Media Stream for MySQL + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Barry Leslie + * + * 2009-07-16 + * + * H&G2JCtL + * + * PBMS interface used to enable engines for use with the PBMS engine. + * + * For an example on how to build this into an engine have a look at the PBXT engine + * in file ha_pbxt.cc. Search for 'PBMS_ENABLED'. + * + */ + +#define PBMS_API pbms_enabled_api + +#include "pbms_enabled.h" +#ifdef DRIZZLED +#include <sys/stat.h> +#include <drizzled/common_includes.h> +#include <drizzled/plugin.h> +#else +#include "mysql_priv.h" +#include <mysql/plugin.h> +#define session_alloc(sess, size) thd_alloc(sess, size); +#define current_session current_thd +#endif + +#define GET_BLOB_FIELD(t, i) (Field_blob *)(t->field[t->s->blob_field[i]]) +#define DB_NAME(f) (f->table->s->db.str) +#define TAB_NAME(f) (*(f->table_name)) + +static PBMS_API pbms_api; + +PBMSEngineRec enabled_engine = { + MS_ENGINE_VERSION +}; + +//==================== +bool pbms_initialize(const char *engine_name, bool isServer, PBMSResultPtr result) +{ + int err; + + strncpy(enabled_engine.ms_engine_name, engine_name, 32); + enabled_engine.ms_internal = isServer; + enabled_engine.ms_engine_name[31] = 0; + + err = pbms_api.registerEngine(&enabled_engine, result); + + return (err == 0); +} + + +//==================== +void pbms_finalize() +{ + pbms_api.deregisterEngine(&enabled_engine); +} + +//==================== +int pbms_write_row_blobs(TABLE *table, uchar *row_buffer, PBMSResultPtr result) +{ + Field_blob *field; + char *blob_rec, *blob; + size_t packlength, i, org_length, length; + char blob_url_buffer[PBMS_BLOB_URL_SIZE]; + int err; + String type_name; + + if (table->s->blob_fields == 0) + return 0; + + for (i= 0; i < table->s->blob_fields; i++) { + field = GET_BLOB_FIELD(table, i); + + // Note: field->type() always returns MYSQL_TYPE_BLOB regardless of the type of BLOB + field->sql_type(type_name); + if (strcasecmp(type_name.c_ptr(), "LongBlob")) + continue; + + // Get the blob record: + blob_rec = (char *)row_buffer + field->offset(field->table->record[0]); + packlength = field->pack_length() - field->table->s->blob_ptr_size; + + memcpy(&blob, blob_rec +packlength, sizeof(char*)); + org_length = field->get_length((uchar *)blob_rec); + + + // Signal PBMS to record a new reference to the BLOB. + // If 'blob' is not a BLOB URL then it will be stored in the repositor as a new BLOB + // and a reference to it will be created. + err = pbms_api.retainBlob(DB_NAME(field), TAB_NAME(field), blob_url_buffer, blob, org_length, field->field_index, result); + if (err) + return err; + + // If the BLOB length changed reset it. + // This will happen if the BLOB data was replaced with a BLOB reference. + length = strlen(blob_url_buffer) +1; + if ((length != org_length) || memcmp(blob_url_buffer, blob, length)) { + if (length != org_length) { + field->store_length((uchar *)blob_rec, packlength, length); + } + + if (length > org_length) { + // This can only happen if the BLOB URL is actually larger than the BLOB itself. + blob = (char *) session_alloc(current_session, length); + memcpy(blob_rec+packlength, &blob, sizeof(char*)); + } + memcpy(blob, blob_url_buffer, length); + } + } + + return 0; +} + +//==================== +int pbms_delete_row_blobs(TABLE *table, const uchar *row_buffer, PBMSResultPtr result) +{ + Field_blob *field; + const char *blob_rec; + char *blob; + size_t packlength, i, length; + int err; + String type_name; + + if (table->s->blob_fields == 0) + return 0; + + for (i= 0; i < table->s->blob_fields; i++) { + field = GET_BLOB_FIELD(table, i); + + // Note: field->type() always returns MYSQL_TYPE_BLOB regardless of the type of BLOB + field->sql_type(type_name); + if (strcasecmp(type_name.c_ptr(), "LongBlob")) + continue; + + // Get the blob record: + blob_rec = (char *)row_buffer + field->offset(field->table->record[0]); + packlength = field->pack_length() - field->table->s->blob_ptr_size; + + length = field->get_length((uchar *)blob_rec); + memcpy(&blob, blob_rec +packlength, sizeof(char*)); + + // Signal PBMS to delete the reference to the BLOB. + err = pbms_api.releaseBlob(DB_NAME(field), TAB_NAME(field), blob, length, result); + if (err) + return err; + } + + return 0; +} + +#define MAX_NAME_SIZE 64 +static void parse_table_path(const char *path, char *db_name, char *tab_name) +{ + const char *ptr = path + strlen(path) -1, *eptr; + int len; + + *db_name = *tab_name = 0; + + while ((ptr > path) && (*ptr != '/'))ptr --; + if (*ptr != '/') + return; + + strncpy(tab_name, ptr+1, MAX_NAME_SIZE); + tab_name[MAX_NAME_SIZE-1] = 0; + eptr = ptr; + ptr--; + + while ((ptr > path) && (*ptr != '/'))ptr --; + if (*ptr != '/') + return; + ptr++; + + len = eptr - ptr; + if (len >= MAX_NAME_SIZE) + len = MAX_NAME_SIZE-1; + + memcpy(db_name, ptr, len); + db_name[len] = 0; + +} + +//==================== +int pbms_rename_table_with_blobs(const char *old_table_path, const char *new_table_path, PBMSResultPtr result) +{ + char o_db_name[MAX_NAME_SIZE], n_db_name[MAX_NAME_SIZE], o_tab_name[MAX_NAME_SIZE], n_tab_name[MAX_NAME_SIZE]; + + parse_table_path(old_table_path, o_db_name, o_tab_name); + parse_table_path(new_table_path, n_db_name, n_tab_name); + + if (strcmp(o_db_name, n_db_name)) { + result->mr_code = MS_ERR_INVALID_OPERATION; + strcpy(result->mr_message, "PBMS does not support renaming tables across databases."); + strcpy(result->mr_stack, "pbms_rename_table_with_blobs()"); + return MS_ERR_INVALID_OPERATION; + } + + + return pbms_api.renameTable(o_db_name, o_tab_name, n_tab_name, result); +} + +//==================== +int pbms_delete_table_with_blobs(const char *table_path, PBMSResultPtr result) +{ + char db_name[MAX_NAME_SIZE], tab_name[MAX_NAME_SIZE]; + + parse_table_path(table_path, db_name, tab_name); + + return pbms_api.dropTable(db_name, tab_name, result); +} + +//==================== +void pbms_completed(TABLE *table, bool ok) +{ + if ((!table) || (table->s->blob_fields != 0)) + pbms_api.completed(ok) ; + + return ; +} + === added file 'storage/pbxt/src/pbms_enabled.h' --- a/storage/pbxt/src/pbms_enabled.h 1970-01-01 00:00:00 +0000 +++ b/storage/pbxt/src/pbms_enabled.h 2009-08-18 07:46:53 +0000 @@ -0,0 +1,110 @@ +/* Copyright (c) 2009 PrimeBase Technologies GmbH, Germany + * + * PrimeBase Media Stream for MySQL + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Barry Leslie + * + * 2009-07-16 + * + * H&G2JCtL + * + * PBMS interface used to enable engines for use with the PBMS engine. + * + * For an example on how to build this into an engine have a look at the PBXT engine + * in file ha_pbxt.cc. Search for 'PBMS_ENABLED'. + * + */ + + +#ifndef __PBMS_ENABLED_H__ +#define __PBMS_ENABLED_H__ + +#include "pbms.h" + +#ifdef DRIZZLED +#include <drizzled/server_includes.h> +#define TABLE Table +#else +#include <mysql_priv.h> +#endif + +/* + * pbms_initialize() should be called from the engines plugIn's 'init()' function. + * The engine_name is the name of your engine, "PBXT" or "InnoDB" for example. + * + * The isServer flag indicates if this entire server is being enabled. This is only + * true if this is being built into the server's handler code above the engine level + * calls. + */ +extern bool pbms_initialize(const char *engine_name, bool isServer, PBMSResultPtr result); + +/* + * pbms_finalize() should be called from the engines plugIn's 'deinit()' function. + */ +extern void pbms_finalize(); + +/* + * pbms_write_row_blobs() should be called from the engine's 'write_row' function. + * It can alter the row data so it must be called before any other function using the row data. + * It should also be called from engine's 'update_row' function for the new row. + * + * pbms_completed() must be called after calling pbms_write_row_blobs() and just before + * returning from write_row() to indicate if the operation completed successfully. + */ +extern int pbms_write_row_blobs(TABLE *table, uchar *buf, PBMSResultPtr result); + +/* + * pbms_delete_row_blobs() should be called from the engine's 'delete_row' function. + * It should also be called from engine's 'update_row' function for the old row. + * + * pbms_completed() must be called after calling pbms_delete_row_blobs() and just before + * returning from delete_row() to indicate if the operation completed successfully. + */ +extern int pbms_delete_row_blobs(TABLE *table, const uchar *buf, PBMSResultPtr result); + +/* + * pbms_rename_table_with_blobs() should be called from the engine's 'rename_table' function. + * + * NOTE: Renaming tables across databases is not supported. + * + * pbms_completed() must be called after calling pbms_rename_table_with_blobs() and just before + * returning from rename_table() to indicate if the operation completed successfully. + */ +extern int pbms_rename_table_with_blobs(const char *old_table_path, const char *new_table_path, PBMSResultPtr result); + +/* + * pbms_delete_table_with_blobs() should be called from the engine's 'delete_table' function. + * + * NOTE: Currently pbms_delete_table_with_blobs() cannot be undone so it should only + * be called after the host engine has performed successfully drop it's table. + * + * pbms_completed() must be called after calling pbms_delete_table_with_blobs() and just before + * returning from delete_table() to indicate if the operation completed successfully. + */ +extern int pbms_delete_table_with_blobs(const char *table_path, PBMSResultPtr result); + +/* + * pbms_completed() must be called to indicate success or failure of a an operation after having + * called pbms_write_row_blobs(), pbms_delete_row_blobs(), pbms_rename_table_with_blobs(), or + * pbms_delete_table_with_blobs(). + * + * pbms_completed() has the effect of committing or rolling back the changes made if the session + * is in 'autocommit' mode. + */ +extern void pbms_completed(TABLE *table, bool ok); + +#endif === modified file 'storage/pbxt/src/pthread_xt.cc' --- a/storage/pbxt/src/pthread_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/pthread_xt.cc 2009-08-18 07:46:53 +0000 @@ -395,20 +395,31 @@ int xt_p_cond_timedwait(xt_cond_type *co int xt_p_join(pthread_t thread, void **value) { - switch (WaitForSingleObject(thread, INFINITE)) { - case WAIT_OBJECT_0: - case WAIT_TIMEOUT: - /* Don't do this! According to the Win docs: - * _endthread automatically closes the thread handle - * (whereas _endthreadex does not). Therefore, when using - * _beginthread and _endthread, do not explicitly close the - * thread handle by calling the Win32 CloseHandle API. - CloseHandle(thread); - */ - break; - case WAIT_FAILED: - return GetLastError(); + DWORD exitcode; + + while(1) { + switch (WaitForSingleObject(thread, 10000)) { + case WAIT_OBJECT_0: + return 0; + case WAIT_TIMEOUT: + /* Don't do this! According to the Win docs: + * _endthread automatically closes the thread handle + * (whereas _endthreadex does not). Therefore, when using + * _beginthread and _endthread, do not explicitly close the + * thread handle by calling the Win32 CloseHandle API. + CloseHandle(thread); + */ + /* This is done so that if the thread was not [yet] in the running + * state when this function was called we won't deadlock here. + */ + if (GetExitCodeThread(thread, &exitcode) && (exitcode == STILL_ACTIVE)) + break; + return 0; + case WAIT_FAILED: + return GetLastError(); + } } + return 0; } === modified file 'storage/pbxt/src/restart_xt.cc' --- a/storage/pbxt/src/restart_xt.cc 2009-04-02 10:03:14 +0000 +++ b/storage/pbxt/src/restart_xt.cc 2009-08-18 07:46:53 +0000 @@ -410,7 +410,7 @@ typedef struct XTOperation { xtLogOffset or_log_offset; } XTOperationRec, *XTOperationPtr; -static int xres_cmp_op_seq(struct XTThread *self __attribute__((unused)), register const void *thunk __attribute__((unused)), register const void *a, register const void *b) +static int xres_cmp_op_seq(struct XTThread *XT_UNUSED(self), register const void *XT_UNUSED(thunk), register const void *a, register const void *b) { xtOpSeqNo lf_op_seq = *((xtOpSeqNo *) a); XTOperationPtr lf_ptr = (XTOperationPtr) b; @@ -480,19 +480,6 @@ static xtBool xres_add_index_entries(XTO return OK; for (idx_cnt=0, ind=tab->tab_dic.dic_keys; idx_cnt<tab->tab_dic.dic_key_count; idx_cnt++, ind++) { - /* - key.sk_on_key = FALSE; - key.sk_key_value.sv_flags = XT_SEARCH_WHOLE_KEY; - key.sk_key_value.sv_rec_id = rec_offset; - key.sk_key_value.sv_key = key.sk_key_buf; - key.sk_key_value.sv_length = myxt_create_key_from_row(*ind, key.sk_key_buf, rec_data, NULL); - if (!xt_idx_search(ot, *ind, &key)) { - ot->ot_err_index_no = (*ind)->mi_index_no; - return FAILED; - } - if (!key.sk_on_key) { - } - */ if (!xt_idx_insert(ot, *ind, row_id, rec_id, rec_data, NULL, TRUE)) { /* Check the error, certain errors are recoverable! */ XTThreadPtr self = xt_get_self(); @@ -509,7 +496,7 @@ static xtBool xres_add_index_entries(XTO /* TODO: Write something to the index header to indicate that * it is corrupted. */ - tab->tab_dic.dic_disable_index = XT_INDEX_CORRUPTED; + xt_tab_disable_index(ot->ot_table, XT_INDEX_CORRUPTED); xt_log_and_clear_exception_ns(); return OK; } @@ -642,6 +629,9 @@ static void xres_apply_change(XTThreadPt xtWord1 *rec_data = NULL; XTTabRecFreeDPtr free_data; + if (tab->tab_dic.dic_key_count == 0) + check_index = FALSE; + switch (record->xl.xl_status_1) { case XT_LOG_ENT_REC_MODIFIED: case XT_LOG_ENT_UPDATE: @@ -651,20 +641,25 @@ static void xres_apply_change(XTThreadPt case XT_LOG_ENT_INSERT_BG: case XT_LOG_ENT_DELETE_BG: rec_id = XT_GET_DISK_4(record->xu.xu_rec_id_4); + + /* This should be done before we apply change to table, as otherwise we lose + * the key value that we need to remove from index + */ + if (check_index && record->xl.xl_status_1 == XT_LOG_ENT_REC_MODIFIED) { + if ((rec_data = xres_load_record(self, ot, rec_id, NULL, 0, rec_buf, tab->tab_dic.dic_ind_cols_req))) + xres_remove_index_entries(ot, rec_id, rec_data); + } + len = (size_t) XT_GET_DISK_2(record->xu.xu_size_2); if (!XT_PWRITE_RR_FILE(ot->ot_rec_file, xt_rec_id_to_rec_offset(tab, rec_id), len, (xtWord1 *) &record->xu.xu_rec_type_1, &ot->ot_thread->st_statistics.st_rec, ot->ot_thread)) xt_throw(self); tab->tab_bytes_to_flush += len; - if (check_index && ot->ot_table->tab_dic.dic_key_count) { + if (check_index) { switch (record->xl.xl_status_1) { case XT_LOG_ENT_DELETE: case XT_LOG_ENT_DELETE_BG: break; - case XT_LOG_ENT_REC_MODIFIED: - if ((rec_data = xres_load_record(self, ot, rec_id, NULL, 0, rec_buf, tab->tab_dic.dic_ind_cols_req))) - xres_remove_index_entries(ot, rec_id, rec_data); - /* No break required: */ default: if ((rec_data = xres_load_record(self, ot, rec_id, &record->xu.xu_rec_type_1, len, rec_buf, tab->tab_dic.dic_ind_cols_req))) { row_id = XT_GET_DISK_4(record->xu.xu_row_id_4); @@ -859,9 +854,6 @@ static void xres_apply_change(XTThreadPt goto do_rec_freed; record_loaded = TRUE; } -#ifdef XT_STREAMING - myxt_release_blobs(ot, rec_data, rec_id); -#endif } if (record->xl.xl_status_1 == XT_LOG_ENT_REC_REMOVED_EXT) { @@ -967,31 +959,12 @@ static void xres_apply_change(XTThreadPt if (check_index) { cols_required = tab->tab_dic.dic_ind_cols_req; -#ifdef XT_STREAMING - if (tab->tab_dic.dic_blob_cols_req > cols_required) - cols_required = tab->tab_dic.dic_blob_cols_req; -#endif if (!(rec_data = xres_load_record(self, ot, rec_id, &record->rb.rb_rec_type_1, rec_size, rec_buf, cols_required))) goto go_on_to_free; record_loaded = TRUE; xres_remove_index_entries(ot, rec_id, rec_data); } -#ifdef XT_STREAMING - if (tab->tab_dic.dic_blob_count) { - if (!record_loaded) { - cols_required = tab->tab_dic.dic_blob_cols_req; - if (!(rec_data = xres_load_record(self, ot, rec_id, &record->rb.rb_rec_type_1, rec_size, rec_buf, cols_required))) - /* [(7)] REMOVE is followed by FREE: - goto get_rec_offset; - */ - goto go_on_to_free; - record_loaded = TRUE; - } - myxt_release_blobs(ot, rec_data, rec_id); - } -#endif - if (data_log_id && data_log_offset && log_over_size) { if (!ot->ot_thread->st_dlog_buf.dlb_delete_log(data_log_id, data_log_offset, log_over_size, tab->tab_id, rec_id, self)) { if (ot->ot_thread->t_exception.e_xt_err != XT_ERR_BAD_EXT_RECORD && @@ -1560,7 +1533,7 @@ static xtBool xres_delete_data_log(XTDat return OK; } -static int xres_comp_flush_tabs(XTThreadPtr self __attribute__((unused)), register const void *thunk __attribute__((unused)), register const void *a, register const void *b) +static int xres_comp_flush_tabs(XTThreadPtr XT_UNUSED(self), register const void *XT_UNUSED(thunk), register const void *a, register const void *b) { xtTableID tab_id = *((xtTableID *) a); XTCheckPointTablePtr cp_tab = (XTCheckPointTablePtr) b; @@ -1868,7 +1841,7 @@ void XTXactRestart::xres_init(XTThreadPt exit_(); } -void XTXactRestart::xres_exit(XTThreadPtr self __attribute__((unused))) +void XTXactRestart::xres_exit(XTThreadPtr XT_UNUSED(self)) { } @@ -2700,7 +2673,7 @@ xtPublic xtBool xt_begin_checkpoint(XTDa XTXactSegPtr seg; seg = &db->db_xn_idx[i]; - XT_XACT_WRITE_LOCK(&seg->xs_tab_lock, self); + XT_XACT_READ_LOCK(&seg->xs_tab_lock, self); for (u_int j=0; j<XT_XN_HASH_TABLE_SIZE; j++) { XTXactDataPtr xact; @@ -2716,7 +2689,7 @@ xtPublic xtBool xt_begin_checkpoint(XTDa xact = xact->xd_next_xact; } } - XT_XACT_UNLOCK(&seg->xs_tab_lock, self); + XT_XACT_UNLOCK(&seg->xs_tab_lock, self, FALSE); } #ifdef TRACE_CHECKPOINT @@ -3201,3 +3174,36 @@ xtPublic void xt_dump_xlogs(XTDatabaseHP done: db->db_xlog.xlog_seq_exit(&seq); } + +/* ---------------------------------------------------------------------- + * D A T A B A S E R E C O V E R Y T H R E A D + */ + +extern XTDatabaseHPtr pbxt_database; + +static void *xn_xres_run_recovery_thread(XTThreadPtr self) +{ + THD *mysql_thread; + + mysql_thread = (THD *)myxt_create_thread(); + + while(!ha_resolve_by_legacy_type(mysql_thread, DB_TYPE_PBXT)) + xt_sleep_milli_second(1); + + xt_open_database(self, mysql_real_data_home, TRUE); + pbxt_database = self->st_database; + xt_heap_reference(self, pbxt_database); + myxt_destroy_thread(mysql_thread, TRUE); + + return NULL; +} + +xtPublic void xt_xres_start_database_recovery(XTThreadPtr self) +{ + char name[PATH_MAX]; + + sprintf(name, "DB-RECOVERY-%s", xt_last_directory_of_path(mysql_real_data_home)); + xt_remove_dir_char(name); + XTThreadPtr thread = xt_create_daemon(self, name); + xt_run_thread(self, thread, xn_xres_run_recovery_thread); +} === modified file 'storage/pbxt/src/restart_xt.h' --- a/storage/pbxt/src/restart_xt.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/restart_xt.h 2009-08-17 11:12:36 +0000 @@ -131,4 +131,6 @@ xtWord8 xt_bytes_since_last_checkpoint(s void xt_print_log_record(xtLogID log, off_t offset, XTXactLogBufferDPtr record); void xt_dump_xlogs(struct XTDatabase *db, xtLogID start_log); +xtPublic void xt_xres_start_database_recovery(XTThreadPtr self, const char *path); + #endif === modified file 'storage/pbxt/src/sortedlist_xt.cc' --- a/storage/pbxt/src/sortedlist_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/sortedlist_xt.cc 2009-08-17 11:12:36 +0000 @@ -234,7 +234,7 @@ xtPublic void xt_sl_delete_item_at(struc XT_MEMMOVE(sl->sl_data, &sl->sl_data[idx * sl->sl_item_size], &sl->sl_data[(idx+1) * sl->sl_item_size], (sl->sl_usage_count-idx) * sl->sl_item_size); } -xtPublic void xt_sl_remove_from_front(struct XTThread *self __attribute__((unused)), XTSortedListPtr sl, size_t items) +xtPublic void xt_sl_remove_from_front(struct XTThread *XT_UNUSED(self), XTSortedListPtr sl, size_t items) { if (sl->sl_usage_count <= items) xt_sl_set_size(sl, 0); === removed file 'storage/pbxt/src/streaming_xt.cc' --- a/storage/pbxt/src/streaming_xt.cc 2009-04-02 10:03:14 +0000 +++ b/storage/pbxt/src/streaming_xt.cc 1970-01-01 00:00:00 +0000 @@ -1,624 +0,0 @@ -/* Copyright (c) 2005 PrimeBase Technologies GmbH - * - * PrimeBase XT - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * 2006-06-07 Paul McCullagh - * - * H&G2JCtL - * - * This file contains PBXT streaming interface. - */ - -#include "xt_config.h" - -#ifdef XT_STREAMING -#include "ha_pbxt.h" - -#include "thread_xt.h" -#include "strutil_xt.h" -#include "table_xt.h" -#include "myxt_xt.h" -#include "xaction_xt.h" -#include "database_xt.h" -#include "streaming_xt.h" - -extern PBMSEngineRec pbxt_engine; - -static PBMS_API pbxt_streaming; - -/* ---------------------------------------------------------------------- - * INIT & EXIT - */ - -xtPublic xtBool xt_init_streaming(void) -{ - XTThreadPtr self = NULL; - int err; - PBMSResultRec result; - - if ((err = pbxt_streaming.registerEngine(&pbxt_engine, &result))) { - xt_logf(XT_CONTEXT, XT_LOG_ERROR, "%s\n", result.mr_message); - return FAILED; - } - return OK; -} - -xtPublic void xt_exit_streaming(void) -{ - pbxt_streaming.deregisterEngine(&pbxt_engine); -} - -/* ---------------------------------------------------------------------- - * UTILITY FUNCTIONS - */ - -static void str_result_to_exception(XTExceptionPtr e, int r, PBMSResultPtr result) -{ - char *str, *end_str; - - e->e_xt_err = r; - e->e_sys_err = result->mr_code; - xt_strcpy(XT_ERR_MSG_SIZE, e->e_err_msg, result->mr_message); - - e->e_source_line = 0; - str = result->mr_stack; - if ((end_str = strchr(str, '('))) { - xt_strcpy_term(XT_MAX_FUNC_NAME_SIZE, e->e_func_name, str, '('); - str = end_str+1; - if ((end_str = strchr(str, ':'))) { - xt_strcpy_term(XT_SOURCE_FILE_NAME_SIZE, e->e_source_file, str, ':'); - str = end_str+1; - if ((end_str = strchr(str, ')'))) { - char number[40]; - - xt_strcpy_term(40, number, str, ')'); - e->e_source_line = atol(number); - str = end_str+1; - if (*str == '\n') - str++; - } - } - } - - if (e->e_source_line == 0) { - *e->e_func_name = 0; - *e->e_source_file = 0; - xt_strcpy(XT_ERR_MSG_SIZE, e->e_catch_trace, result->mr_stack); - } - else - xt_strcpy(XT_ERR_MSG_SIZE, e->e_catch_trace, str); -} - -static void str_exception_to_result(XTExceptionPtr e, PBMSResultPtr result) -{ - int len; - - if (e->e_sys_err) - result->mr_code = e->e_sys_err; - else - result->mr_code = e->e_xt_err; - xt_strcpy(MS_RESULT_MESSAGE_SIZE, result->mr_message, e->e_err_msg); - xt_strcpy(MS_RESULT_STACK_SIZE, result->mr_stack, e->e_func_name); - xt_strcat(MS_RESULT_STACK_SIZE, result->mr_stack, "("); - xt_strcat(MS_RESULT_STACK_SIZE, result->mr_stack, e->e_source_file); - xt_strcat(MS_RESULT_STACK_SIZE, result->mr_stack, ":"); - xt_strcati(MS_RESULT_STACK_SIZE, result->mr_stack, (int) e->e_source_line); - xt_strcat(MS_RESULT_STACK_SIZE, result->mr_stack, ")"); - len = strlen(result->mr_stack); - if (strncmp(result->mr_stack, e->e_catch_trace, len) == 0) - xt_strcat(MS_RESULT_STACK_SIZE, result->mr_stack, e->e_catch_trace + len); - else { - xt_strcat(MS_RESULT_STACK_SIZE, result->mr_stack, "\n"); - xt_strcat(MS_RESULT_STACK_SIZE, result->mr_stack, e->e_catch_trace); - } -} - -static XTIndexPtr str_find_index(XTTableHPtr tab, u_int *col_list, u_int col_cnt) -{ - u_int i, j; - XTIndexPtr *ind; /* MySQL/PBXT key description */ - - ind = tab->tab_dic.dic_keys; - for (i=0; i<tab->tab_dic.dic_key_count; i++) { - if ((*ind)->mi_seg_count == col_cnt) { - for (j=0; j<(*ind)->mi_seg_count; j++) { - if ((*ind)->mi_seg[j].col_idx != col_list[j]) - goto loop; - } - return *ind; - } - - loop: - ind++; - } - return NULL; -} - -static XTThreadPtr str_set_current_thread(THD *thd, PBMSResultPtr result) -{ - XTThreadPtr self; - XTExceptionRec e; - - if (!(self = xt_ha_set_current_thread(thd, &e))) { - str_exception_to_result(&e, result); - return NULL; - } - return self; -} - -/* ---------------------------------------------------------------------- - * BLOB STREAMING INTERFACE - */ - -static void pbxt_close_conn(void *thread) -{ - xt_ha_close_connection((THD *) thread); -} - -static int pbxt_open_table(void *thread, const char *table_url, void **open_table, PBMSResultPtr result) -{ - THD *thd = (THD *) thread; - XTThreadPtr self; - XTTableHPtr tab = NULL; - XTOpenTablePtr ot = NULL; - int err = MS_OK; - - if (!(self = str_set_current_thread(thd, result))) - return MS_ERR_ENGINE; - - try_(a) { - xt_ha_open_database_of_table(self, (XTPathStrPtr) table_url); - if (!(tab = xt_use_table(self, (XTPathStrPtr) table_url, FALSE, TRUE, NULL))) { - err = MS_ERR_UNKNOWN_TABLE; - goto done; - } - if (!(ot = xt_open_table(tab))) - throw_(); - ot->ot_thread = self; - done:; - } - catch_(a) { - str_exception_to_result(&self->t_exception, result); - err = MS_ERR_ENGINE; - } - cont_(a); - if (tab) - xt_heap_release(self, tab); - *open_table = ot; - return err; -} - -static void pbxt_close_table(void *thread, void *open_table_ptr) -{ - THD *thd = (THD *) thread; - volatile XTThreadPtr self, new_self = NULL; - XTOpenTablePtr ot = (XTOpenTablePtr) open_table_ptr; - XTExceptionRec e; - - if (thd) { - if (!(self = xt_ha_set_current_thread(thd, &e))) { - xt_log_exception(NULL, &e, XT_LOG_DEFAULT); - return; - } - } - else if (!(self = xt_get_self())) { - if (!(new_self = xt_create_thread("TempForClose", FALSE, TRUE, &e))) { - xt_log_exception(NULL, &e, XT_LOG_DEFAULT); - return; - } - self = new_self; - } - - ot->ot_thread = self; - try_(a) { - xt_close_table(ot, TRUE, FALSE); - } - catch_(a) { - xt_log_and_clear_exception(self); - } - cont_(a); - if (new_self) - xt_free_thread(self); -} - -static int pbxt_lock_table(void *thread, int *xact, void *open_table, int lock_type, PBMSResultPtr result) -{ - THD *thd = (THD *) thread; - XTThreadPtr self; - XTOpenTablePtr ot = (XTOpenTablePtr) open_table; - int err = MS_OK; - - if (!(self = str_set_current_thread(thd, result))) - return MS_ERR_ENGINE; - - if (lock_type != MS_LOCK_NONE) { - try_(a) { - xt_ha_open_database_of_table(self, ot->ot_table->tab_name); - ot->ot_thread = self; - } - catch_(a) { - str_exception_to_result(&self->t_exception, result); - err = MS_ERR_ENGINE; - } - cont_(a); - } - - if (!err && *xact == MS_XACT_BEGIN) { - if (self->st_xact_data) - *xact = MS_XACT_NONE; - else { - if (xt_xn_begin(self)) { - *xact = MS_XACT_COMMIT; - } - else { - str_exception_to_result(&self->t_exception, result); - err = MS_ERR_ENGINE; - } - } - } - - return err; -} - -static int pbxt_unlock_table(void *thread, int xact, void *open_table __attribute__((unused)), PBMSResultPtr result) -{ - THD *thd = (THD *) thread; - XTThreadPtr self = xt_ha_thd_to_self(thd); - int err = MS_OK; - - if (xact == MS_XACT_COMMIT) { - if (!xt_xn_commit(self)) { - str_exception_to_result(&self->t_exception, result); - err = MS_ERR_ENGINE; - } - } - else if (xact == MS_XACT_ROLLBACK) { - xt_xn_rollback(self); - } - - return err; -} - -static int pbxt_send_blob(void *thread, void *open_table, const char *blob_column, const char *blob_url_p, void *stream, PBMSResultPtr result) -{ - THD *thd = (THD *) thread; - XTThreadPtr self = xt_ha_thd_to_self(thd); - XTOpenTablePtr ot = (XTOpenTablePtr) open_table; - int err = MS_OK; - u_int blob_col_idx, col_idx; - char col_name[XT_IDENTIFIER_NAME_SIZE]; - XTStringBufferRec value; - u_int col_list[XT_MAX_COLS_PER_INDEX]; - u_int col_cnt; - char col_names[XT_ERR_MSG_SIZE - 200]; - XTIdxSearchKeyRec search_key; - XTIndexPtr ind; - char *blob_data; - size_t blob_len; - const char *blob_url = blob_url_p; - - memset(&value, 0, sizeof(value)); - - *col_names = 0; - - ot->ot_thread = self; - try_(a) { - if (ot->ot_row_wbuf_size < ot->ot_table->tab_dic.dic_mysql_buf_size) { - xt_realloc(self, (void **) &ot->ot_row_wbuffer, ot->ot_table->tab_dic.dic_mysql_buf_size); - ot->ot_row_wbuf_size = ot->ot_table->tab_dic.dic_mysql_buf_size; - } - - xt_strcpy_url(XT_IDENTIFIER_NAME_SIZE, col_name, blob_column); - if (!myxt_find_column(ot, &blob_col_idx, col_name)) - xt_throw_tabcolerr(XT_CONTEXT, XT_ERR_COLUMN_NOT_FOUND, ot->ot_table->tab_name, blob_column); - - /* Prepare a row for the condition: */ - const char *ptr; - - col_cnt = 0; - while (*blob_url) { - ptr = xt_strchr(blob_url, '='); - xt_strncpy_url(XT_IDENTIFIER_NAME_SIZE, col_name, blob_url, (size_t) (ptr - blob_url)); - if (!myxt_find_column(ot, &col_idx, col_name)) - xt_throw_tabcolerr(XT_CONTEXT, XT_ERR_COLUMN_NOT_FOUND, ot->ot_table->tab_name, col_name); - if (*col_names) - xt_strcat(sizeof(col_names), col_names, ", "); - xt_strcat(sizeof(col_names), col_names, col_name); - blob_url = ptr; - if (*blob_url == '=') - blob_url++; - ptr = xt_strchr(blob_url, '&'); - value.sb_len = 0; - xt_sb_concat_url_len(self, &value, blob_url, (size_t) (ptr - blob_url)); - blob_url = ptr; - if (*blob_url == '&') - blob_url++; - if (!myxt_set_column(ot, (char *) ot->ot_row_rbuffer, col_idx, value.sb_cstring, value.sb_len)) - xt_throw_tabcolerr(XT_CONTEXT, XT_ERR_CONVERSION, ot->ot_table->tab_name, col_name); - if (col_cnt < XT_MAX_COLS_PER_INDEX) { - col_list[col_cnt] = col_idx; - col_cnt++; - } - } - - /* Find a matching index: */ - if (!(ind = str_find_index(ot->ot_table, col_list, col_cnt))) - xt_throw_ixterr(XT_CONTEXT, XT_ERR_NO_MATCHING_INDEX, col_names); - - search_key.sk_key_value.sv_flags = 0; - search_key.sk_key_value.sv_rec_id = 0; - search_key.sk_key_value.sv_row_id = 0; - search_key.sk_key_value.sv_key = search_key.sk_key_buf; - search_key.sk_key_value.sv_length = myxt_create_key_from_row(ind, search_key.sk_key_buf, ot->ot_row_rbuffer, NULL); - search_key.sk_on_key = FALSE; - - if (!xt_idx_search(ot, ind, &search_key)) - xt_throw(self); - - if (!ot->ot_curr_rec_id) - xt_throw_taberr(XT_CONTEXT, XT_ERR_NO_ROWS, ot->ot_table->tab_name); - - while (ot->ot_curr_rec_id) { - if (!search_key.sk_on_key) - xt_throw_taberr(XT_CONTEXT, XT_ERR_NO_ROWS, ot->ot_table->tab_name); - - retry: - /* X TODO - Check if the write buffer is big enough here! */ - switch (xt_tab_read_record(ot, ot->ot_row_wbuffer)) { - case FALSE: - if (xt_idx_next(ot, ind, &search_key)) - break; - case XT_ERR: - xt_throw(self); - case XT_NEW: - if (xt_idx_match_search(ot, ind, &search_key, ot->ot_row_wbuffer, XT_S_MODE_MATCH)) - goto success; - if (!xt_idx_next(ot, ind, &search_key)) - xt_throw(self); - break; - case XT_RETRY: - goto retry; - default: - goto success; - } - } - - success: - myxt_get_column_data(ot, (char *) ot->ot_row_wbuffer, blob_col_idx, &blob_data, &blob_len); - - /* - * Write the content length, then write the HTTP - * header, and then the content. - */ - err = pbxt_streaming.setContentLength(stream, blob_len, result); - if (!err) - err = pbxt_streaming.writeHead(stream, result); - if (!err) - err = pbxt_streaming.writeStream(stream, (void *) blob_data, blob_len, result); - } - catch_(a) { - str_exception_to_result(&self->t_exception, result); - if (result->mr_code == XT_ERR_NO_ROWS) - err = MS_ERR_NOT_FOUND; - else - err = MS_ERR_ENGINE; - } - cont_(a); - xt_sb_set_size(NULL, &value, 0); - return err; -} - -int pbxt_lookup_ref(void *thread, void *open_table, unsigned short col_index, PBMSEngineRefPtr eng_ref, PBMSFieldRefPtr field_ref, PBMSResultPtr result) -{ - THD *thd = (THD *) thread; - XTThreadPtr self = xt_ha_thd_to_self(thd); - XTOpenTablePtr ot = (XTOpenTablePtr) open_table; - int err = MS_OK; - u_int i, len; - char *data; - XTIndexPtr ind = NULL; - - ot->ot_thread = self; - if (ot->ot_row_wbuf_size < ot->ot_table->tab_dic.dic_mysql_buf_size) { - xt_realloc(self, (void **) &ot->ot_row_wbuffer, ot->ot_table->tab_dic.dic_mysql_buf_size); - ot->ot_row_wbuf_size = ot->ot_table->tab_dic.dic_mysql_buf_size; - } - - ot->ot_curr_rec_id = (xtRecordID) XT_GET_DISK_8(eng_ref->er_data); - switch (xt_tab_dirty_read_record(ot, ot->ot_row_wbuffer)) { - case FALSE: - err = MS_ERR_ENGINE; - break; - default: - break; - } - - if (err) { - str_exception_to_result(&self->t_exception, result); - goto exit; - } - - myxt_get_column_name(ot, col_index, PBMS_FIELD_COL_SIZE, field_ref->fr_column); - - for (i=0; i<ot->ot_table->tab_dic.dic_key_count; i++) { - ind = ot->ot_table->tab_dic.dic_keys[i]; - if (ind->mi_flags & (HA_UNIQUE_CHECK | HA_NOSAME)) - break; - } - - if (ind) { - len = 0; - data = field_ref->fr_cond; - for (i=0; i<ind->mi_seg_count; i++) { - if (i > 0) { - xt_strcat(PBMS_FIELD_COND_SIZE, data, "&"); - len = strlen(data); - } - myxt_get_column_name(ot, ind->mi_seg[i].col_idx, PBMS_FIELD_COND_SIZE - len, data + len); - len = strlen(data); - xt_strcat(PBMS_FIELD_COND_SIZE, data, "="); - len = strlen(data); - myxt_get_column_as_string(ot, (char *) ot->ot_row_wbuffer, ind->mi_seg[i].col_idx, PBMS_FIELD_COND_SIZE - len, data + len); - len = strlen(data); - } - } - else - xt_strcpy(PBMS_FIELD_COND_SIZE, field_ref->fr_cond, "*no unique key*"); - - exit: - return err; -} - -PBMSEngineRec pbxt_engine = { - MS_ENGINE_VERSION, - 0, - FALSE, - "PBXT", - NULL, - pbxt_close_conn, - pbxt_open_table, - pbxt_close_table, - pbxt_lock_table, - pbxt_unlock_table, - pbxt_send_blob, - pbxt_lookup_ref -}; - -/* ---------------------------------------------------------------------- - * CALL IN FUNCTIONS - */ - -xtPublic void xt_pbms_close_all_tables(const char *table_url) -{ - pbxt_streaming.closeAllTables(table_url); -} - -xtPublic xtBool xt_pbms_close_connection(void *thd, XTExceptionPtr e) -{ - PBMSResultRec result; - int err; - - err = pbxt_streaming.closeConn(thd, &result); - if (err) { - str_result_to_exception(e, err, &result); - return FAILED; - } - return OK; -} - -xtPublic xtBool xt_pbms_open_table(void **open_table, char *table_path) -{ - PBMSResultRec result; - int err; - - err = pbxt_streaming.openTable(open_table, table_path, &result); - if (err) { - XTThreadPtr thread = xt_get_self(); - - str_result_to_exception(&thread->t_exception, err, &result); - return FAILED; - } - return OK; -} - -xtPublic void xt_pbms_close_table(void *open_table) -{ - PBMSResultRec result; - int err; - - err = pbxt_streaming.closeTable(open_table, &result); - if (err) { - XTThreadPtr thread = xt_get_self(); - - str_result_to_exception(&thread->t_exception, err, &result); - xt_log_exception(thread, &thread->t_exception, XT_LOG_DEFAULT); - } -} - -xtPublic xtBool xt_pbms_use_blob(void *open_table, char **ret_blob_url, char *blob_url, unsigned short col_index) -{ - PBMSResultRec result; - int err; - - err = pbxt_streaming.useBlob(open_table, ret_blob_url, blob_url, col_index, &result); - if (err) { - XTThreadPtr thread = xt_get_self(); - - str_result_to_exception(&thread->t_exception, err, &result); - return FAILED; - } - return OK; -} - -xtPublic xtBool xt_pbms_retain_blobs(void *open_table, PBMSEngineRefPtr eng_ref) -{ - PBMSResultRec result; - int err; - - err = pbxt_streaming.retainBlobs(open_table, eng_ref, &result); - if (err) { - XTThreadPtr thread = xt_get_self(); - - str_result_to_exception(&thread->t_exception, err, &result); - return FAILED; - } - return OK; -} - -xtPublic void xt_pbms_release_blob(void *open_table, char *blob_url, unsigned short col_index, PBMSEngineRefPtr eng_ref) -{ - PBMSResultRec result; - int err; - - err = pbxt_streaming.releaseBlob(open_table, blob_url, col_index, eng_ref, &result); - if (err) { - XTThreadPtr thread = xt_get_self(); - - str_result_to_exception(&thread->t_exception, err, &result); - xt_log_exception(thread, &thread->t_exception, XT_LOG_DEFAULT); - } -} - -xtPublic void xt_pbms_drop_table(const char *table_path) -{ - PBMSResultRec result; - int err; - - err = pbxt_streaming.dropTable(table_path, &result); - if (err) { - XTThreadPtr thread = xt_get_self(); - - str_result_to_exception(&thread->t_exception, err, &result); - xt_log_exception(thread, &thread->t_exception, XT_LOG_DEFAULT); - } -} - -xtPublic void xt_pbms_rename_table(const char *from_table, const char *to_table) -{ - PBMSResultRec result; - int err; - - err = pbxt_streaming.renameTable(from_table, to_table, &result); - if (err) { - XTThreadPtr thread = xt_get_self(); - - str_result_to_exception(&thread->t_exception, err, &result); - xt_log_exception(thread, &thread->t_exception, XT_LOG_DEFAULT); - } -} - -#endif // XT_STREAMING === removed file 'storage/pbxt/src/streaming_xt.h' --- a/storage/pbxt/src/streaming_xt.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/streaming_xt.h 1970-01-01 00:00:00 +0000 @@ -1,46 +0,0 @@ -/* Copyright (c) 2005 PrimeBase Technologies GmbH - * - * PrimeBase XT - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * 2006-06-07 Paul McCullagh - * - * H&G2JCtL - * - * This file contains PBXT streaming interface. - */ - -#ifndef __streaming_xt_h__ -#define __streaming_xt_h__ - -#include "xt_defs.h" -#define PBMS_API pbms_api_PBXT -#include "pbms.h" - -xtBool xt_init_streaming(void); -void xt_exit_streaming(void); - -void xt_pbms_close_all_tables(const char *table_url); -xtBool xt_pbms_close_connection(void *thd, XTExceptionPtr e); -xtBool xt_pbms_open_table(void **open_table, char *table_path); -void xt_pbms_close_table(void *open_table); -xtBool xt_pbms_use_blob(void *open_table, char **ret_blob_url, char *blob_url, unsigned short col_index); -xtBool xt_pbms_retain_blobs(void *open_table, PBMSEngineRefPtr eng_ref); -void xt_pbms_release_blob(void *open_table, char *blob_url, unsigned short col_index, PBMSEngineRefPtr eng_ref); -void xt_pbms_drop_table(const char *table_path); -void xt_pbms_rename_table(const char *from_table, const char *to_table); - -#endif === modified file 'storage/pbxt/src/strutil_xt.cc' --- a/storage/pbxt/src/strutil_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/strutil_xt.cc 2009-08-18 07:46:53 +0000 @@ -365,9 +365,10 @@ xtPublic void xt_int8_to_byte_size(xtInt sprintf(string, "%s %s (%"PRId64" bytes)", val_str, unit, value); } +/* Version number must also be set in configure.in! */ xtPublic c_char *xt_get_version(void) { - return "1.0.08 RC"; + return "1.0.08c RC"; } /* Copy and URL decode! */ === modified file 'storage/pbxt/src/systab_xt.cc' --- a/storage/pbxt/src/systab_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/systab_xt.cc 2009-08-17 11:12:36 +0000 @@ -217,7 +217,7 @@ bool XTLocationTable::seqScanNext(char * uint32 len; Field *curr_field; byte *save; - MY_BITMAP *save_write_set; + MX_BITMAP *save_write_set; last_access = CS_GET_DISK_4(blob->rb_last_access_4); last_ref = CS_GET_DISK_4(blob->rb_last_ref_4); @@ -336,7 +336,6 @@ bool XTLocationTable::seqScanNext(char * table->write_set = save_write_set; return true; #endif - return false; } void XTLocationTable::loadRow(char *buf, xtWord4 row_id) @@ -345,7 +344,7 @@ void XTLocationTable::loadRow(char *buf, Field *curr_field; XTTablePathPtr tp_ptr; byte *save; - MY_BITMAP *save_write_set; + MX_BITMAP *save_write_set; /* ASSERT_COLUMN_MARKED_FOR_WRITE is failing when * I use store()!?? @@ -386,7 +385,7 @@ void XTLocationTable::loadRow(char *buf, table->write_set = save_write_set; } -xtWord4 XTLocationTable::seqScanPos(xtWord1 *buf __attribute__((unused))) +xtWord4 XTLocationTable::seqScanPos(xtWord1 *XT_UNUSED(buf)) { return lt_index-1; } @@ -451,7 +450,7 @@ bool XTStatisticsTable::seqScanNext(char void XTStatisticsTable::loadRow(char *buf, xtWord4 rec_id) { TABLE *table = ost_my_table; - MY_BITMAP *save_write_set; + MX_BITMAP *save_write_set; Field *curr_field; byte *save; const char *stat_name; @@ -503,7 +502,7 @@ void XTStatisticsTable::loadRow(char *bu table->write_set = save_write_set; } -xtWord4 XTStatisticsTable::seqScanPos(xtWord1 *buf __attribute__((unused))) +xtWord4 XTStatisticsTable::seqScanPos(xtWord1 *XT_UNUSED(buf)) { return tt_index-1; } @@ -531,14 +530,14 @@ void st_path_to_table_name(size_t size, *str = '.'; } -void XTSystemTableShare::startUp(XTThreadPtr self __attribute__((unused))) +void XTSystemTableShare::startUp(XTThreadPtr XT_UNUSED(self)) { thr_lock_init(&sys_location_lock); thr_lock_init(&sys_statistics_lock); sys_lock_inited = TRUE; } -void XTSystemTableShare::shutDown(XTThreadPtr self __attribute__((unused))) +void XTSystemTableShare::shutDown(XTThreadPtr XT_UNUSED(self)) { if (sys_lock_inited) { thr_lock_delete(&sys_location_lock); @@ -588,7 +587,7 @@ bool XTSystemTableShare::doesSystemTable return false; } -void XTSystemTableShare::createSystemTables(XTThreadPtr self __attribute__((unused)), XTDatabaseHPtr db __attribute__((unused))) +void XTSystemTableShare::createSystemTables(XTThreadPtr XT_UNUSED(self), XTDatabaseHPtr XT_UNUSED(db)) { int i = 0; === modified file 'storage/pbxt/src/systab_xt.h' --- a/storage/pbxt/src/systab_xt.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/systab_xt.h 2009-08-17 11:12:36 +0000 @@ -85,15 +85,15 @@ public: virtual bool use() { return true; } virtual bool unuse() { return true; } virtual bool seqScanInit() { return true; } - virtual bool seqScanNext(char *buf __attribute__((unused)), bool *eof) { + virtual bool seqScanNext(char *XT_UNUSED(buf), bool *eof) { *eof = true; return false; } virtual int getRefLen() { return 4; } - virtual xtWord4 seqScanPos(xtWord1 *buf __attribute__((unused))) { + virtual xtWord4 seqScanPos(xtWord1 *XT_UNUSED(buf)) { return 0; } - virtual bool seqScanRead(xtWord4 rec_id __attribute__((unused)), char *buf __attribute__((unused))) { + virtual bool seqScanRead(xtWord4 XT_UNUSED(rec_id), char *XT_UNUSED(buf)) { return true; } === modified file 'storage/pbxt/src/tabcache_xt.cc' --- a/storage/pbxt/src/tabcache_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/tabcache_xt.cc 2009-08-17 11:12:36 +0000 @@ -26,6 +26,10 @@ #include "xt_config.h" +#ifdef DRIZZLED +#include <bitset> +#endif + #include <signal.h> #include "pthread_xt.h" @@ -63,7 +67,7 @@ xtPublic void xt_tc_init(XTThreadPtr sel for (u_int i=0; i<XT_TC_SEGMENT_COUNT; i++) { xt_tab_cache.tcm_segment[i].tcs_cache_in_use = 0; xt_tab_cache.tcm_segment[i].tcs_hash_table = (XTTabCachePagePtr *) xt_calloc(self, xt_tab_cache.tcm_hash_size * sizeof(XTTabCachePagePtr)); - xt_rwmutex_init_with_autoname(self, &xt_tab_cache.tcm_segment[i].tcs_lock); + TAB_CAC_INIT_LOCK(self, &xt_tab_cache.tcm_segment[i].tcs_lock); } xt_init_mutex_with_autoname(self, &xt_tab_cache.tcm_lock); @@ -97,7 +101,7 @@ xtPublic void xt_tc_exit(XTThreadPtr sel xt_free(self, xt_tab_cache.tcm_segment[i].tcs_hash_table); xt_tab_cache.tcm_segment[i].tcs_hash_table = NULL; - xt_rwmutex_free(self, &xt_tab_cache.tcm_segment[i].tcs_lock); + TAB_CAC_FREE_LOCK(self, &xt_tab_cache.tcm_segment[i].tcs_lock); } } @@ -213,7 +217,7 @@ xtBool XTTabCache::xt_tc_write(XT_ROW_RE page->tcp_dirty = TRUE; ASSERT_NS(page->tcp_db_id == tci_table->tab_db->db_id && page->tcp_tab_id == tci_table->tab_id); *op_seq = tci_table->tab_seq.ts_set_op_seq(page); - xt_rwmutex_unlock(&seg->tcs_lock, thread->t_id); + TAB_CAC_UNLOCK(&seg->tcs_lock, thread->t_id); return OK; } @@ -269,21 +273,36 @@ xtBool XTTabCache::xt_tc_write_cond(XTTh page->tcp_dirty = TRUE; ASSERT(page->tcp_db_id == tci_table->tab_db->db_id && page->tcp_tab_id == tci_table->tab_id); *op_seq = tci_table->tab_seq.ts_set_op_seq(page); - xt_rwmutex_unlock(&seg->tcs_lock, self->t_id); + TAB_CAC_UNLOCK(&seg->tcs_lock, self->t_id); return TRUE; no_change: - xt_rwmutex_unlock(&seg->tcs_lock, self->t_id); + TAB_CAC_UNLOCK(&seg->tcs_lock, self->t_id); return FALSE; } xtBool XTTabCache::xt_tc_read(XT_ROW_REC_FILE_PTR file, xtRefID ref_id, size_t size, xtWord1 *data, XTThreadPtr thread) { +#ifdef XT_USE_ROW_REC_MMAP_FILES return tc_read_direct(file, ref_id, size, data, thread); +#else + size_t offset; + XTTabCachePagePtr page; + XTTabCacheSegPtr seg; + + if (!tc_fetch(file, ref_id, &seg, &page, &offset, TRUE, thread)) + return FAILED; + /* A read must be completely on a page: */ + ASSERT_NS(offset + size <= tci_page_size); + memcpy(data, page->tcp_data + offset, size); + TAB_CAC_UNLOCK(&seg->tcs_lock, thread->t_id); + return OK; +#endif } xtBool XTTabCache::xt_tc_read_4(XT_ROW_REC_FILE_PTR file, xtRefID ref_id, xtWord4 *value, XTThreadPtr thread) { +#ifdef XT_USE_ROW_REC_MMAP_FILES register u_int page_idx; register XTTabCachePagePtr page; register XTTabCacheSegPtr seg; @@ -300,7 +319,7 @@ xtBool XTTabCache::xt_tc_read_4(XT_ROW_R seg = &dcg->tcm_segment[hash_idx & XT_TC_SEGMENT_MASK]; hash_idx = (hash_idx >> XT_TC_SEGMENT_SHIFTS) % dcg->tcm_hash_size; - xt_rwmutex_slock(&seg->tcs_lock, thread->t_id); + TAB_CAC_READ_LOCK(&seg->tcs_lock, thread->t_id); page = seg->tcs_hash_table[hash_idx]; while (page) { if (page->tcp_page_idx == page_idx && page->tcp_file_id == file->fr_id) { @@ -311,53 +330,60 @@ xtBool XTTabCache::xt_tc_read_4(XT_ROW_R ASSERT_NS(offset + 4 <= this->tci_page_size); buffer = page->tcp_data + offset; *value = XT_GET_DISK_4(buffer); - xt_rwmutex_unlock(&seg->tcs_lock, thread->t_id); + TAB_CAC_UNLOCK(&seg->tcs_lock, thread->t_id); return OK; } page = page->tcp_next; } - xt_rwmutex_unlock(&seg->tcs_lock, thread->t_id); + TAB_CAC_UNLOCK(&seg->tcs_lock, thread->t_id); -#ifdef XT_USE_ROW_REC_MMAP_FILES return xt_pread_fmap_4(file, address, value, &thread->st_statistics.st_rec, thread); #else - xtWord1 data[4]; + size_t offset; + XTTabCachePagePtr page; + XTTabCacheSegPtr seg; + xtWord1 *data; - if (!XT_PREAD_RR_FILE(file, address, 4, 4, data, NULL, &thread->st_statistics.st_rec, thread)) + if (!tc_fetch(file, ref_id, &seg, &page, &offset, TRUE, thread)) return FAILED; + /* A read must be completely on a page: */ + ASSERT_NS(offset + 4 <= tci_page_size); + data = page->tcp_data + offset; *value = XT_GET_DISK_4(data); + TAB_CAC_UNLOCK(&seg->tcs_lock, thread->t_id); return OK; #endif } -xtBool XTTabCache::xt_tc_get_page(XT_ROW_REC_FILE_PTR file, xtRefID ref_id, XTTabCachePagePtr *ret_page, size_t *offset, XTThreadPtr thread) +xtBool XTTabCache::xt_tc_get_page(XT_ROW_REC_FILE_PTR file, xtRefID ref_id, xtBool load, XTTabCachePagePtr *ret_page, size_t *offset, XTThreadPtr thread) { XTTabCachePagePtr page; XTTabCacheSegPtr seg; -#ifdef XT_SEQ_SCAN_FROM_MEMORY - if (!tc_fetch_direct(file, ref_id, &seg, &page, offset, thread)) - return FAILED; - if (!seg) { - *ret_page = NULL; - return OK; + if (load) { + if (!tc_fetch(file, ref_id, &seg, &page, offset, TRUE, thread)) + return FAILED; + } + else { + if (!tc_fetch_direct(file, ref_id, &seg, &page, offset, thread)) + return FAILED; + if (!seg) { + *ret_page = NULL; + return OK; + } } -#else - if (!tc_fetch(file, ref_id, &seg, &page, offset, TRUE, thread)) - return FAILED; -#endif page->tcp_lock_count++; - xt_rwmutex_unlock(&seg->tcs_lock, thread->t_id); + TAB_CAC_UNLOCK(&seg->tcs_lock, thread->t_id); *ret_page = page; return OK; } -void XTTabCache::xt_tc_release_page(XT_ROW_REC_FILE_PTR file __attribute__((unused)), XTTabCachePagePtr page, XTThreadPtr thread) +void XTTabCache::xt_tc_release_page(XT_ROW_REC_FILE_PTR XT_UNUSED(file), XTTabCachePagePtr page, XTThreadPtr thread) { XTTabCacheSegPtr seg; seg = &xt_tab_cache.tcm_segment[page->tcp_seg]; - xt_rwmutex_xlock(&seg->tcs_lock, thread->t_id); + TAB_CAC_WRITE_LOCK(&seg->tcs_lock, thread->t_id); #ifdef DEBUG XTTabCachePagePtr lpage, ppage; @@ -379,7 +405,7 @@ void XTTabCache::xt_tc_release_page(XT_R if (page->tcp_lock_count > 0) page->tcp_lock_count--; - xt_rwmutex_unlock(&seg->tcs_lock, thread->t_id); + TAB_CAC_UNLOCK(&seg->tcs_lock, thread->t_id); } xtBool XTTabCache::xt_tc_read_page(XT_ROW_REC_FILE_PTR file, xtRefID ref_id, xtWord1 *data, XTThreadPtr thread) @@ -412,7 +438,7 @@ xtBool XTTabCache::tc_read_direct(XT_ROW seg = &dcg->tcm_segment[hash_idx & XT_TC_SEGMENT_MASK]; hash_idx = (hash_idx >> XT_TC_SEGMENT_SHIFTS) % dcg->tcm_hash_size; - xt_rwmutex_slock(&seg->tcs_lock, thread->t_id); + TAB_CAC_READ_LOCK(&seg->tcs_lock, thread->t_id); page = seg->tcs_hash_table[hash_idx]; while (page) { if (page->tcp_page_idx == page_idx && page->tcp_file_id == file->fr_id) { @@ -421,12 +447,12 @@ xtBool XTTabCache::tc_read_direct(XT_ROW offset = (ref_id % this->tci_rows_per_page) * this->tci_rec_size; ASSERT_NS(offset + size <= this->tci_page_size); memcpy(data, page->tcp_data + offset, size); - xt_rwmutex_unlock(&seg->tcs_lock, thread->t_id); + TAB_CAC_UNLOCK(&seg->tcs_lock, thread->t_id); return OK; } page = page->tcp_next; } - xt_rwmutex_unlock(&seg->tcs_lock, thread->t_id); + TAB_CAC_UNLOCK(&seg->tcs_lock, thread->t_id); if (!XT_PREAD_RR_FILE(file, address, size, 0, data, &red_size, &thread->st_statistics.st_rec, thread)) return FAILED; memset(data + red_size, 0, size - red_size); @@ -450,7 +476,7 @@ xtBool XTTabCache::tc_fetch_direct(XT_RO seg = &dcg->tcm_segment[hash_idx & XT_TC_SEGMENT_MASK]; hash_idx = (hash_idx >> XT_TC_SEGMENT_SHIFTS) % dcg->tcm_hash_size; - xt_rwmutex_xlock(&seg->tcs_lock, thread->t_id); + TAB_CAC_WRITE_LOCK(&seg->tcs_lock, thread->t_id); page = seg->tcs_hash_table[hash_idx]; while (page) { if (page->tcp_page_idx == page_idx && page->tcp_file_id == file->fr_id) { @@ -460,7 +486,7 @@ xtBool XTTabCache::tc_fetch_direct(XT_RO } page = page->tcp_next; } - xt_rwmutex_unlock(&seg->tcs_lock, thread->t_id); + TAB_CAC_UNLOCK(&seg->tcs_lock, thread->t_id); *ret_seg = NULL; *ret_page = NULL; return OK; @@ -492,7 +518,7 @@ xtBool XTTabCache::tc_fetch(XT_ROW_REC_F seg = &dcg->tcm_segment[hash_idx & XT_TC_SEGMENT_MASK]; hash_idx = (hash_idx >> XT_TC_SEGMENT_SHIFTS) % dcg->tcm_hash_size; - xt_rwmutex_slock(&seg->tcs_lock, thread->t_id); + TAB_CAC_READ_LOCK(&seg->tcs_lock, thread->t_id); page = seg->tcs_hash_table[hash_idx]; while (page) { if (page->tcp_page_idx == page_idx && page->tcp_file_id == file->fr_id) { @@ -528,7 +554,7 @@ xtBool XTTabCache::tc_fetch(XT_ROW_REC_F } page = page->tcp_next; } - xt_rwmutex_unlock(&seg->tcs_lock, thread->t_id); + TAB_CAC_UNLOCK(&seg->tcs_lock, thread->t_id); /* Page not found, allocate a new page: */ size_t page_size = offsetof(XTTabCachePageRec, tcp_data) + this->tci_page_size; @@ -674,7 +700,7 @@ xtBool XTTabCache::tc_fetch(XT_ROW_REC_F #endif /* Add the page to the cache! */ - xt_rwmutex_xlock(&seg->tcs_lock, thread->t_id); + TAB_CAC_WRITE_LOCK(&seg->tcs_lock, thread->t_id); page = seg->tcs_hash_table[hash_idx]; while (page) { if (page->tcp_page_idx == page_idx && page->tcp_file_id == file->fr_id) { @@ -898,11 +924,11 @@ static size_t tabc_free_page(XTThreadPtr } seg = &dcg->tcm_segment[page->tcp_seg]; - xt_rwmutex_xlock(&seg->tcs_lock, self->t_id); + TAB_CAC_WRITE_LOCK(&seg->tcs_lock, self->t_id); if (page->tcp_dirty) { if (!was_dirty) { - xt_rwmutex_unlock(&seg->tcs_lock, self->t_id); + TAB_CAC_UNLOCK(&seg->tcs_lock, self->t_id); goto retry_2; } @@ -923,7 +949,7 @@ static size_t tabc_free_page(XTThreadPtr XTDatabaseHPtr db = tab->tab_db; rewait: - xt_rwmutex_unlock(&seg->tcs_lock, self->t_id); + TAB_CAC_UNLOCK(&seg->tcs_lock, self->t_id); /* Flush the log, in case this is holding up the * writer! @@ -963,7 +989,7 @@ static size_t tabc_free_page(XTThreadPtr db->db_wr_freeer_waiting = FALSE; freer_(); // xt_unlock_mutex(&db->db_wr_lock) - xt_rwmutex_xlock(&seg->tcs_lock, self->t_id); + TAB_CAC_WRITE_LOCK(&seg->tcs_lock, self->t_id); if (XTTableSeq::xt_op_is_before(tab->tab_head_op_seq, page->tcp_op_seq)) goto rewait; } @@ -988,11 +1014,11 @@ static size_t tabc_free_page(XTThreadPtr */ if ((page = page->tcp_mr_used)) { page_cnt++; - xt_rwmutex_unlock(&seg->tcs_lock, self->t_id); + TAB_CAC_UNLOCK(&seg->tcs_lock, self->t_id); goto retry_2; } } - xt_rwmutex_unlock(&seg->tcs_lock, self->t_id); + TAB_CAC_UNLOCK(&seg->tcs_lock, self->t_id); dcg->tcm_free_try_count++; /* Starting to spin, free the threads: */ @@ -1047,7 +1073,7 @@ static size_t tabc_free_page(XTThreadPtr seg->tcs_cache_in_use -= freed_space; xt_free_ns(page); - xt_rwmutex_unlock(&seg->tcs_lock, self->t_id); + TAB_CAC_UNLOCK(&seg->tcs_lock, self->t_id); self->st_statistics.st_rec_cache_frees++; dcg->tcm_free_try_count = 0; return freed_space; @@ -1156,7 +1182,7 @@ static void *tabc_fr_run_thread(XTThread return NULL; } -static void tabc_fr_free_thread(XTThreadPtr self, void *data __attribute__((unused))) +static void tabc_fr_free_thread(XTThreadPtr self, void *XT_UNUSED(data)) { if (xt_tab_cache.tcm_freeer_thread) { xt_lock_mutex(self, &xt_tab_cache.tcm_freeer_lock); @@ -1238,7 +1264,7 @@ xtPublic void xt_load_pages(XTThreadPtr while (rec_id<tab->tab_row_eof_id) { if (!tab->tab_rows.tc_fetch(ot->ot_row_file, rec_id, &seg, &page, &poffset, TRUE, self)) xt_throw(self); - xt_rwmutex_unlock(&seg->tcs_lock, self->t_id); + TAB_CAC_UNLOCK(&seg->tcs_lock, self->t_id); rec_id += tab->tab_rows.tci_rows_per_page; } @@ -1246,7 +1272,7 @@ xtPublic void xt_load_pages(XTThreadPtr while (rec_id<tab->tab_rec_eof_id) { if (!tab->tab_recs.tc_fetch(ot->ot_rec_file, rec_id, &seg, &page, &poffset, TRUE, self)) xt_throw(self); - xt_rwmutex_unlock(&seg->tcs_lock, self->t_id); + TAB_CAC_UNLOCK(&seg->tcs_lock, self->t_id); rec_id += tab->tab_recs.tci_rows_per_page; } } === modified file 'storage/pbxt/src/tabcache_xt.h' --- a/storage/pbxt/src/tabcache_xt.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/tabcache_xt.h 2009-08-17 11:12:36 +0000 @@ -125,11 +125,11 @@ typedef struct XTTableSeq { xt_init_mutex_with_autoname(self, &ts_ns_lock); } - void xt_op_seq_set(XTThreadPtr self __attribute__((unused)), xtOpSeqNo n) { + void xt_op_seq_set(XTThreadPtr XT_UNUSED(self), xtOpSeqNo n) { ts_next_seq = n; } - void xt_op_seq_exit(XTThreadPtr self __attribute__((unused))) { + void xt_op_seq_exit(XTThreadPtr XT_UNUSED(self)) { xt_free_mutex(&ts_ns_lock); } @@ -150,12 +150,50 @@ typedef struct XTTableSeq { #endif } XTTableSeqRec, *XTTableSeqPtr; +#ifdef XT_NO_ATOMICS +#define TAB_CAC_USE_PTHREAD_RW +#else +//#define TAB_CAC_USE_RWMUTEX +//#define TAB_CAC_USE_PTHREAD_RW +//#define IDX_USE_SPINXSLOCK +#define TAB_CAC_USE_XSMUTEX +#endif + +#ifdef TAB_CAC_USE_XSMUTEX +#define TAB_CAC_LOCK_TYPE XTXSMutexRec +#define TAB_CAC_INIT_LOCK(s, i) xt_xsmutex_init_with_autoname(s, i) +#define TAB_CAC_FREE_LOCK(s, i) xt_xsmutex_free(s, i) +#define TAB_CAC_READ_LOCK(i, o) xt_xsmutex_slock(i, o) +#define TAB_CAC_WRITE_LOCK(i, o) xt_xsmutex_xlock(i, o) +#define TAB_CAC_UNLOCK(i, o) xt_xsmutex_unlock(i, o) +#elif defined(TAB_CAC_USE_PTHREAD_RW) +#define TAB_CAC_LOCK_TYPE xt_rwlock_type +#define TAB_CAC_INIT_LOCK(s, i) xt_init_rwlock(s, i) +#define TAB_CAC_FREE_LOCK(s, i) xt_free_rwlock(i) +#define TAB_CAC_READ_LOCK(i, o) xt_slock_rwlock_ns(i) +#define TAB_CAC_WRITE_LOCK(i, o) xt_xlock_rwlock_ns(i) +#define TAB_CAC_UNLOCK(i, o) xt_unlock_rwlock_ns(i) +#elif defined(TAB_CAC_USE_RWMUTEX) +#define TAB_CAC_LOCK_TYPE XTRWMutexRec +#define TAB_CAC_INIT_LOCK(s, i) xt_rwmutex_init_with_autoname(s, i) +#define TAB_CAC_FREE_LOCK(s, i) xt_rwmutex_free(s, i) +#define TAB_CAC_READ_LOCK(i, o) xt_rwmutex_slock(i, o) +#define TAB_CAC_WRITE_LOCK(i, o) xt_rwmutex_xlock(i, o) +#define TAB_CAC_UNLOCK(i, o) xt_rwmutex_unlock(i, o) +#elif defined(TAB_CAC_USE_SPINXSLOCK) +#define TAB_CAC_LOCK_TYPE XTSpinXSLockRec +#define TAB_CAC_INIT_LOCK(s, i) xt_spinxslock_init_with_autoname(s, i) +#define TAB_CAC_FREE_LOCK(s, i) xt_spinxslock_free(s, i) +#define TAB_CAC_READ_LOCK(i, o) xt_spinxslock_slock(i, o) +#define TAB_CAC_WRITE_LOCK(i, o) xt_spinxslock_xlock(i, o) +#define TAB_CAC_UNLOCK(i, o) xt_spinxslock_unlock(i, o) +#endif + /* A disk cache segment. The cache is divided into a number of segments * to improve concurrency. */ typedef struct XTTabCacheSeg { - XTRWMutexRec tcs_lock; /* The cache segment read/write lock. */ - //xt_cond_type tcs_cond; + TAB_CAC_LOCK_TYPE tcs_lock; /* The cache segment read/write lock. */ XTTabCachePagePtr *tcs_hash_table; size_t tcs_cache_in_use; } XTTabCacheSegRec, *XTTabCacheSegPtr; @@ -220,7 +258,7 @@ public: xtBool xt_tc_read(XT_ROW_REC_FILE_PTR file, xtRefID ref_id, size_t size, xtWord1 *data, XTThreadPtr thread); xtBool xt_tc_read_4(XT_ROW_REC_FILE_PTR file, xtRefID ref_id, xtWord4 *data, XTThreadPtr thread); xtBool xt_tc_read_page(XT_ROW_REC_FILE_PTR file, xtRefID ref_id, xtWord1 *data, XTThreadPtr thread); - xtBool xt_tc_get_page(XT_ROW_REC_FILE_PTR file, xtRefID ref_id, XTTabCachePagePtr *page, size_t *offset, XTThreadPtr thread); + xtBool xt_tc_get_page(XT_ROW_REC_FILE_PTR file, xtRefID ref_id, xtBool load, XTTabCachePagePtr *page, size_t *offset, XTThreadPtr thread); void xt_tc_release_page(XT_ROW_REC_FILE_PTR file, XTTabCachePagePtr page, XTThreadPtr thread); xtBool tc_fetch(XT_ROW_REC_FILE_PTR file, xtRefID ref_id, XTTabCacheSegPtr *ret_seg, XTTabCachePagePtr *ret_page, size_t *offset, xtBool read, XTThreadPtr thread); === modified file 'storage/pbxt/src/table_xt.cc' --- a/storage/pbxt/src/table_xt.cc 2009-04-02 20:36:52 +0000 +++ b/storage/pbxt/src/table_xt.cc 2009-08-18 07:46:53 +0000 @@ -35,7 +35,7 @@ #include <drizzled/common.h> #include <mysys/thr_lock.h> #include <drizzled/dtcollation.h> -#include <drizzled/handlerton.h> +#include <drizzled/plugin/storage_engine.h> #else #include "mysql_priv.h" #endif @@ -47,9 +47,6 @@ #include "myxt_xt.h" #include "cache_xt.h" #include "trace_xt.h" -#ifdef XT_STREAMING -#include "streaming_xt.h" -#endif #include "index_xt.h" #include "restart_xt.h" #include "systab_xt.h" @@ -293,17 +290,17 @@ static void tab_get_row_file_name(char * sprintf(table_name, "%s-%lu.xtr", name, (u_long) tab_id); } -static void tab_get_data_file_name(char *table_name, char *name, xtTableID tab_id __attribute__((unused))) +static void tab_get_data_file_name(char *table_name, char *name, xtTableID XT_UNUSED(tab_id)) { sprintf(table_name, "%s.xtd", name); } -static void tab_get_index_file_name(char *table_name, char *name, xtTableID tab_id __attribute__((unused))) +static void tab_get_index_file_name(char *table_name, char *name, xtTableID XT_UNUSED(tab_id)) { sprintf(table_name, "%s.xti", name); } -static void tab_free_by_id(XTThreadPtr self __attribute__((unused)), void *thunk __attribute__((unused)), void *item) +static void tab_free_by_id(XTThreadPtr self, void *XT_UNUSED(thunk), void *item) { XTTableEntryPtr te_ptr = (XTTableEntryPtr) item; @@ -315,7 +312,7 @@ static void tab_free_by_id(XTThreadPtr s te_ptr->te_table = NULL; } -static int tab_comp_by_id(XTThreadPtr self __attribute__((unused)), register const void *thunk __attribute__((unused)), register const void *a, register const void *b) +static int tab_comp_by_id(XTThreadPtr XT_UNUSED(self), register const void *XT_UNUSED(thunk), register const void *a, register const void *b) { xtTableID te_id = *((xtTableID *) a); XTTableEntryPtr te_ptr = (XTTableEntryPtr) b; @@ -327,14 +324,14 @@ static int tab_comp_by_id(XTThreadPtr se return 1; } -static void tab_free_path(XTThreadPtr self __attribute__((unused)), void *thunk __attribute__((unused)), void *item) +static void tab_free_path(XTThreadPtr self, void *XT_UNUSED(thunk), void *item) { XTTablePathPtr tp_ptr = *((XTTablePathPtr *) item); xt_free(self, tp_ptr); } -static int tab_comp_path(XTThreadPtr self __attribute__((unused)), register const void *thunk __attribute__((unused)), register const void *a, register const void *b) +static int tab_comp_path(XTThreadPtr XT_UNUSED(self), register const void *XT_UNUSED(thunk), register const void *a, register const void *b) { char *path = (char *) a; XTTablePathPtr tp_ptr = *((XTTablePathPtr *) b); @@ -398,7 +395,7 @@ xtPublic xtBool xt_describe_tables_next( return_(TRUE); } -xtPublic void xt_describe_tables_exit(XTThreadPtr self __attribute__((unused)), XTTableDescPtr td) +xtPublic void xt_describe_tables_exit(XTThreadPtr XT_UNUSED(self), XTTableDescPtr td) { if (td->td_open_dir) xt_dir_close(NULL, td->td_open_dir); @@ -410,8 +407,11 @@ xtPublic void xt_tab_init_db(XTThreadPtr { XTTableDescRec desc; XTTableEntryRec te_tab; + XTTableEntryPtr te_ptr; XTTablePathPtr db_path; + char pbuf[PATH_MAX]; int len; + u_int edx; enter_(); pushr_(xt_tab_exit_db, db); @@ -425,7 +425,6 @@ xtPublic void xt_tab_init_db(XTThreadPtr if (db->db_multi_path) { XTOpenFilePtr of; char *buffer, *ptr, *path; - char pbuf[PATH_MAX]; xt_strcpy(PATH_MAX, pbuf, db->db_main_path); xt_add_location_file(PATH_MAX, pbuf); @@ -490,6 +489,27 @@ xtPublic void xt_tab_init_db(XTThreadPtr } freer_(); // xt_describe_tables_exit(&desc) + /* + * The purpose of this code is to ensure that all tables are opened and cached, + * which is actually only required if tables have foreign key references. + * + * In other words, a side affect of this code is that FK references between tables + * are registered, and checked. + * + * Unfortunately we don't know if a table is referenced by a FK, so we have to open + * all tables. + * + * Cannot open tables in the loop above because db->db_table_by_id which is built + * above is used by xt_use_table_no_lock() + */ + xt_enum_tables_init(&edx); + while ((te_ptr = xt_enum_tables_next(self, db, &edx))) { + xt_strcpy(PATH_MAX, pbuf, te_ptr->te_tab_path->tp_path); + xt_add_dir_char(PATH_MAX, pbuf); + xt_strcat(PATH_MAX, pbuf, te_ptr->te_tab_name); + xt_heap_release(self, xt_use_table_no_lock(self, db, (XTPathStrPtr)pbuf, FALSE, FALSE, NULL, NULL)); + } + popr_(); // Discard xt_tab_exit_db(db) exit_(); } @@ -605,8 +625,9 @@ xtPublic void xt_tab_exit_db(XTThreadPtr } } -static void tab_check_table(XTThreadPtr self __attribute__((unused)), XTTableHPtr tab __attribute__((unused))) +static void tab_check_table(XTThreadPtr self, XTTableHPtr XT_UNUSED(tab)) { + (void) self; enter_(); exit_(); } @@ -661,7 +682,7 @@ xtPublic void xt_enum_tables_init(u_int *edx = 0; } -xtPublic XTTableEntryPtr xt_enum_tables_next(XTThreadPtr self __attribute__((unused)), XTDatabaseHPtr db, u_int *edx) +xtPublic XTTableEntryPtr xt_enum_tables_next(XTThreadPtr XT_UNUSED(self), XTDatabaseHPtr db, u_int *edx) { XTTableEntryPtr en_ptr; @@ -727,6 +748,12 @@ static xtBool tab_find_table(XTThreadPtr return FALSE; } +xtPublic void xt_tab_disable_index(XTTableHPtr tab, u_int ind_error) +{ + tab->tab_dic.dic_disable_index = ind_error; + xt_tab_set_table_repair_pending(tab); +} + xtPublic void xt_tab_set_index_error(XTTableHPtr tab) { switch (tab->tab_dic.dic_disable_index) { @@ -803,22 +830,39 @@ static void tab_load_index_header(XTThre tab->tab_index_page_size = XT_GET_DISK_4(index_fmt->if_page_size_4); } +#ifdef XT_USE_LAZY_DELETE + if (tab->tab_dic.dic_index_ver <= XT_IND_NO_LAZY_DELETE) + tab->tab_dic.dic_no_lazy_delete = TRUE; + else + tab->tab_dic.dic_no_lazy_delete = FALSE; +#else + tab->tab_dic.dic_no_lazy_delete = TRUE; +#endif + /* Incorrect version of index is handled by allowing a sequential scan, but no index access. * Recovery with the wrong index type will not recover the indexes, a REPAIR TABLE * will be required! */ if (tab->tab_dic.dic_index_ver != XT_IND_CURRENT_VERSION) { - if (tab->tab_dic.dic_index_ver != XT_IND_CURRENT_VERSION) - tab->tab_dic.dic_disable_index = XT_INDEX_TOO_OLD; - else - tab->tab_dic.dic_disable_index = XT_INDEX_TOO_NEW; + switch (tab->tab_dic.dic_index_ver) { + case XT_IND_NO_LAZY_DELETE: + case XT_IND_LAZY_DELETE_OK: + /* I can handle this type of index. */ + break; + default: + if (tab->tab_dic.dic_index_ver < XT_IND_CURRENT_VERSION) + xt_tab_disable_index(tab, XT_INDEX_TOO_OLD); + else + xt_tab_disable_index(tab, XT_INDEX_TOO_NEW); + break; + } } else if (tab->tab_index_page_size != XT_INDEX_PAGE_SIZE) - tab->tab_dic.dic_disable_index = XT_INDEX_BAD_BLOCK; + xt_tab_disable_index(tab, XT_INDEX_BAD_BLOCK); } else { memset(tab->tab_index_head, 0, XT_INDEX_HEAD_SIZE); - tab->tab_dic.dic_disable_index = XT_INDEX_MISSING; + xt_tab_disable_index(tab, XT_INDEX_MISSING); tab->tab_index_header_size = XT_INDEX_HEAD_SIZE; tab->tab_index_page_size = XT_INDEX_PAGE_SIZE; tab->tab_dic.dic_index_ver = 0; @@ -1089,6 +1133,8 @@ static int tab_new_handle(XTThreadPtr se xt_heap_set_release_callback(self, tab, tab_onrelease); + tab->tab_repair_pending = xt_tab_is_table_repair_pending(tab); + popr_(); // Discard xt_heap_release(tab) xt_ht_put(self, db->db_tables, tab); @@ -1216,11 +1262,6 @@ static XTOpenTablePoolPtr tab_lock_table return_(NULL); } -#ifdef XT_STREAMING - /* Tell PBMS to close all open tables of this sort: */ - xt_pbms_close_all_tables(name->ps_path); -#endif - /* Wait for all open tables to close: */ xt_db_wait_for_open_tables(self, table_pool); @@ -1297,9 +1338,6 @@ xtPublic void xt_create_table(XTThreadPt /* Remove the PBMS table: */ ASSERT(xt_get_self() == self); -#ifdef XT_STREAMING - xt_pbms_drop_table(name->ps_path); -#endif /* Remove the table from the directory. It will get a new * ID so the handle in the directory will no longer be valid. @@ -1572,7 +1610,7 @@ xtPublic void xt_create_table(XTThreadPt exit_(); } -xtPublic void xt_drop_table(XTThreadPtr self, XTPathStrPtr tab_name) +xtPublic void xt_drop_table(XTThreadPtr self, XTPathStrPtr tab_name, xtBool drop_db) { XTDatabaseHPtr db = self->st_database; XTOpenTablePoolPtr table_pool; @@ -1596,8 +1634,16 @@ xtPublic void xt_drop_table(XTThreadPtr tab_id = tab->tab_id; /* tab is not null if returned table_pool is not null */ /* check if other tables refer this */ if (!self->st_ignore_fkeys) - can_drop = tab->tab_dic.dic_table->checkCanDrop(); + can_drop = tab->tab_dic.dic_table->checkCanDrop(drop_db); } +#ifdef DRIZZLED + /* See the comment in ha_pbxt::delete_table regarding different implmentation of DROP TABLE + * in MySQL and Drizzle + */ + else { + xt_throw_xterr(XT_CONTEXT, XT_ERR_TABLE_NOT_FOUND); + } +#endif if (can_drop) { if (tab_id) { @@ -1614,9 +1660,6 @@ xtPublic void xt_drop_table(XTThreadPtr tab_delete_table_files(self, tab_name, tab_id); ASSERT(xt_get_self() == self); -#ifdef XT_STREAMING - xt_pbms_drop_table(tab_name->ps_path); -#endif if ((te_ptr = (XTTableEntryPtr) xt_sl_find(self, db->db_table_by_id, &tab_id))) { tab_remove_table_path(self, db, te_ptr->te_tab_path); xt_sl_delete(self, db->db_table_by_id, &tab_id); @@ -1733,6 +1776,7 @@ xtPublic void xt_check_table(XTThreadPtr u_llong max_comp_rec_len = 0; size_t rec_size; size_t row_size; + u_llong ext_data_len = 0; #if defined(DUMP_CHECK_TABLE) || defined(CHECK_TABLE_STATS) printf("\nCHECK TABLE: %s\n", tab->tab_name->ps_path); @@ -1832,6 +1876,7 @@ xtPublic void xt_check_table(XTThreadPtr printf("record-X "); #endif alloc_rec_count++; + ext_data_len += XT_GET_DISK_4(rec_buf->re_log_dat_siz_4); row_size = XT_GET_DISK_4(rec_buf->re_log_dat_siz_4) + ot->ot_rec_size - XT_REC_EXT_HEADER_SIZE; alloc_rec_bytes += row_size; if (!min_comp_rec_len || row_size < min_comp_rec_len) @@ -1887,6 +1932,9 @@ xtPublic void xt_check_table(XTThreadPtr } #ifdef CHECK_TABLE_STATS + if (!tab->tab_dic.dic_rec_fixed) + printf("Extendend data length = %llu\n", ext_data_len); + if (alloc_rec_count) { printf("Minumum comp. rec. len. = %llu\n", (u_llong) min_comp_rec_len); printf("Average comp. rec. len. = %llu\n", (u_llong) ((double) alloc_rec_bytes / (double) alloc_rec_count + (double) 0.5)); @@ -2055,6 +2103,8 @@ xtPublic void xt_rename_table(XTThreadPt popr_(); // Discard xt_free(te_new_name); tab = xt_use_table_no_lock(self, db, new_name, FALSE, FALSE, &dic, NULL); + /* All renamed tables are considered repaired! */ + xt_tab_table_repaired(tab); xt_heap_release(self, tab); freer_(); // myxt_free_dictionary(&dic) @@ -2306,6 +2356,9 @@ xtPublic XTOpenTablePtr tab_open_table(X return NULL; memset(ot, 0, offsetof(XTOpenTableRec, ot_ind_wbuf)); + ot->ot_seq_page = NULL; + ot->ot_seq_data = NULL; + self = xt_get_self(); try_(a) { xt_heap_reference(self, tab); @@ -3353,6 +3406,16 @@ xtPublic int xt_tab_dirty_read_record(re return OK; } +#ifdef XT_USE_ROW_REC_MMAP_FILES +/* Loading into cache is not required, + * Instead we copy the memory map to load the + * data. + */ +#define TAB_ROW_LOAD_CACHE FALSE +#else +#define TAB_ROW_LOAD_CACHE TRUE +#endif + /* * Pull the entire row pointer file into memory. */ @@ -3376,7 +3439,7 @@ xtPublic void xt_tab_load_row_pointers(X end_offset = xt_row_id_to_row_offset(tab, eof_rec_id); rec_id = 1; while (rec_id < eof_rec_id) { - if (!tab->tab_rows.xt_tc_get_page(ot->ot_row_file, rec_id, &page, &poffset, self)) + if (!tab->tab_rows.xt_tc_get_page(ot->ot_row_file, rec_id, TAB_ROW_LOAD_CACHE, &page, &poffset, self)) xt_throw(self); if (page) tab->tab_rows.xt_tc_release_page(ot->ot_row_file, page, self); @@ -3392,7 +3455,7 @@ xtPublic void xt_tab_load_row_pointers(X XT_LOCK_MEMORY_PTR(buff_ptr, ot->ot_row_file, offset, tfer, &self->st_statistics.st_rec, self); if (buff_ptr) { memcpy(buffer, buff_ptr, tfer); - XT_UNLOCK_MEMORY_PTR(ot->ot_row_file, self); + XT_UNLOCK_MEMORY_PTR(ot->ot_row_file, buff_ptr, TRUE, self); } } rec_id += tab->tab_rows.tci_rows_per_page; @@ -3521,7 +3584,7 @@ static void tab_restore_exception(XTExce * FALSE if the record has already been freed. * */ -xtPublic int xt_tab_remove_record(XTOpenTablePtr ot, xtRecordID rec_id, xtWord1 *rec_data, xtRecordID *prev_var_id, xtBool clean_delete, xtRowID row_id, xtXactID xn_id __attribute__((unused))) +xtPublic int xt_tab_remove_record(XTOpenTablePtr ot, xtRecordID rec_id, xtWord1 *rec_data, xtRecordID *prev_var_id, xtBool clean_delete, xtRowID row_id, xtXactID XT_UNUSED(xn_id)) { register XTTableHPtr tab = ot->ot_table; size_t rec_size; @@ -3664,49 +3727,6 @@ xtPublic int xt_tab_remove_record(XTOpen } } -#ifdef XT_STREAMING - if (tab->tab_dic.dic_blob_count) { - /* If the record contains any LONGBLOB then check how much - * space we need. - */ - size_t blob_size; - - switch (old_rec_type) { - case XT_TAB_STATUS_DELETE: - case XT_TAB_STATUS_DEL_CLEAN: - break; - case XT_TAB_STATUS_FIXED: - case XT_TAB_STATUS_FIX_CLEAN: - /* Should not be the case, record with LONGBLOB can never be fixed! */ - break; - case XT_TAB_STATUS_VARIABLE: - case XT_TAB_STATUS_VAR_CLEAN: - cols_req = tab->tab_dic.dic_blob_cols_req; - cols_in_buffer = cols_req; - blob_size = myxt_load_row_length(ot, rec_size - XT_REC_FIX_HEADER_SIZE, ot->ot_row_rbuffer + XT_REC_FIX_HEADER_SIZE, &cols_in_buffer); - if (cols_in_buffer < cols_req) - blob_size = tab->tab_dic.dic_rec_size; - else - blob_size += XT_REC_FIX_HEADER_SIZE; - if (blob_size > rec_size) - rec_size = blob_size; - break; - case XT_TAB_STATUS_EXT_DLOG: - case XT_TAB_STATUS_EXT_CLEAN: - cols_req = tab->tab_dic.dic_blob_cols_req; - cols_in_buffer = cols_req; - blob_size = myxt_load_row_length(ot, rec_size - XT_REC_EXT_HEADER_SIZE, ot->ot_row_rbuffer + XT_REC_EXT_HEADER_SIZE, &cols_in_buffer); - if (cols_in_buffer < cols_req) - blob_size = tab->tab_dic.dic_rec_size; - else - blob_size += XT_REC_EXT_HEADER_SIZE; - if (blob_size > rec_size) - rec_size = blob_size; - break; - } - } -#endif - set_removed: if (XT_REC_IS_EXT_DLOG(old_rec_type)) { /* {LOCK-EXT-REC} Lock, and read again to make sure that the @@ -3810,7 +3830,7 @@ static xtRowID tab_new_row(XTOpenTablePt xt_unlock_mutex_ns(&tab->tab_row_lock); return 0; } - xt_rwmutex_unlock(&seg->tcs_lock, ot->ot_thread->t_id); + TAB_CAC_UNLOCK(&seg->tcs_lock, ot->ot_thread->t_id); } tab->tab_row_eof_id++; } @@ -4343,15 +4363,6 @@ xtPublic xtBool xt_tab_new_record(XTOpen xtRowID row_id; u_int idx_cnt = 0; XTIndexPtr *ind; -#ifdef XT_STREAMING - void *pbms_table; - - /* PBMS: Reference BLOBs!? */ - if (tab->tab_dic.dic_blob_count) { - if (!myxt_use_blobs(ot, &pbms_table, rec_buf)) - return FAILED; - } -#endif if (!myxt_store_row(ot, &rec_info, (char *) rec_buf)) goto failed_0; @@ -4386,17 +4397,6 @@ xtPublic xtBool xt_tab_new_record(XTOpen } } -#ifdef XT_STREAMING - /* Reference the BLOBs in the row: */ - if (tab->tab_dic.dic_blob_count) { - if (!myxt_retain_blobs(ot, pbms_table, rec_info.ri_rec_id)) { - pbms_table = NULL; - goto failed_2; - } - pbms_table = NULL; - } -#endif - /* Do the foreign key stuff: */ if (ot->ot_table->tab_dic.dic_table->dt_fkeys.size() > 0) { if (!ot->ot_table->tab_dic.dic_table->insertRow(ot, rec_buf)) @@ -4417,10 +4417,6 @@ xtPublic xtBool xt_tab_new_record(XTOpen tab_free_row_on_fail(ot, tab, row_id); failed_0: -#ifdef XT_STREAMING - if (tab->tab_dic.dic_blob_count && pbms_table) - myxt_unuse_blobs(ot, pbms_table); -#endif return FAILED; } @@ -4524,15 +4520,6 @@ static xtBool tab_overwrite_record(XTOpe xtLogOffset log_offset; xtBool prev_ext_rec; -#ifdef XT_STREAMING - void *pbms_table; - - if (tab->tab_dic.dic_blob_count) { - if (!myxt_use_blobs(ot, &pbms_table, after_buf)) - return FAILED; - } -#endif - if (!myxt_store_row(ot, &rec_info, (char *) after_buf)) goto failed_0; @@ -4596,16 +4583,6 @@ static xtBool tab_overwrite_record(XTOpe if (prev_ext_rec) tab_free_ext_record_on_fail(ot, rec_id, &prev_rec_head, TRUE); -#ifdef XT_STREAMING - if (tab->tab_dic.dic_blob_count) { - /* Retain the BLOBs new record: */ - if (!myxt_retain_blobs(ot, pbms_table, rec_id)) - return FAILED; - /* Release the BLOBs in the old record: */ - myxt_release_blobs(ot, before_buf, rec_id); - } -#endif - return OK; failed_2: @@ -4648,11 +4625,6 @@ static xtBool tab_overwrite_record(XTOpe tab_free_ext_record_on_fail(ot, rec_id, &prev_rec_head, TRUE); failed_0: -#ifdef XT_STREAMING - /* Unuse the BLOBs of the new record: */ - if (tab->tab_dic.dic_blob_count && pbms_table) - myxt_unuse_blobs(ot, pbms_table); -#endif return FAILED; } @@ -4666,10 +4638,6 @@ xtPublic xtBool xt_tab_update_record(XTO u_int idx_cnt = 0; XTIndexPtr *ind; -#ifdef XT_STREAMING - void *pbms_table; -#endif - /* * Originally only the flag ot->ot_curr_updated was checked, and if it was on, then * tab_overwrite_record() was called, but this caused crashes in some cases like: @@ -4709,14 +4677,6 @@ xtPublic xtBool xt_tab_update_record(XTO row_id = ot->ot_curr_row_id; self = ot->ot_thread; -#ifdef XT_STREAMING - /* PBMS: Reference BLOBs!? */ - if (tab->tab_dic.dic_blob_count) { - if (!myxt_use_blobs(ot, &pbms_table, after_buf)) - return FAILED; - } -#endif - if (!myxt_store_row(ot, &rec_info, (char *) after_buf)) goto failed_0; @@ -4766,17 +4726,6 @@ xtPublic xtBool xt_tab_update_record(XTO } } -#ifdef XT_STREAMING - /* Reference the BLOBs in the row: */ - if (tab->tab_dic.dic_blob_count) { - if (!myxt_retain_blobs(ot, pbms_table, rec_info.ri_rec_id)) { - pbms_table = NULL; - goto failed_2; - } - pbms_table = NULL; - } -#endif - if (ot->ot_table->tab_dic.dic_table->dt_trefs || ot->ot_table->tab_dic.dic_table->dt_fkeys.size() > 0) { if (!ot->ot_table->tab_dic.dic_table->updateRow(ot, before_buf, after_buf)) goto failed_2; @@ -4793,10 +4742,6 @@ xtPublic xtBool xt_tab_update_record(XTO XT_TAB_ROW_UNLOCK(&tab->tab_row_rwlock[row_id % XT_ROW_RWLOCKS], ot->ot_thread); failed_0: -#ifdef XT_STREAMING - if (tab->tab_dic.dic_blob_count && pbms_table) - myxt_unuse_blobs(ot, pbms_table); -#endif return FAILED; } @@ -4906,6 +4851,7 @@ xtPublic xtBool xt_tab_seq_init(XTOpenTa register XTTableHPtr tab = ot->ot_table; ot->ot_seq_page = NULL; + ot->ot_seq_data = NULL; ot->ot_on_page = FALSE; ot->ot_seq_offset = 0; @@ -4958,6 +4904,7 @@ xtPublic void xt_tab_seq_reset(XTOpenTab ot->ot_seq_rec_id = 0; ot->ot_seq_eof_id = 0; ot->ot_seq_page = NULL; + ot->ot_seq_data = NULL; ot->ot_on_page = FALSE; ot->ot_seq_offset = 0; } @@ -4970,23 +4917,40 @@ xtPublic void xt_tab_seq_exit(XTOpenTabl tab->tab_recs.xt_tc_release_page(ot->ot_rec_file, ot->ot_seq_page, ot->ot_thread); ot->ot_seq_page = NULL; } + if (ot->ot_seq_data) + XT_UNLOCK_MEMORY_PTR(ot->ot_rec_file, ot->ot_seq_data, TRUE, ot->ot_thread); ot->ot_on_page = FALSE; } +#ifdef XT_USE_ROW_REC_MMAP_FILES +#define TAB_SEQ_LOAD_CACHE FALSE +#else +#ifdef XT_SEQ_SCAN_LOADS_CACHE +#define TAB_SEQ_LOAD_CACHE TRUE +#else +#define TAB_SEQ_LOAD_CACHE FALSE +#endif +#endif + xtPublic xtBool xt_tab_seq_next(XTOpenTablePtr ot, xtWord1 *buffer, xtBool *eof) { register XTTableHPtr tab = ot->ot_table; register size_t rec_size = tab->tab_dic.dic_rec_size; xtWord1 *buff_ptr; xtRecordID new_rec_id; - xtBool ptr_locked; xtRecordID invalid_rec = 0; - XTTabRecHeadDRec rec_head; next_page: if (!ot->ot_on_page) { - if (!(ot->ot_on_page = tab->tab_recs.xt_tc_get_page(ot->ot_rec_file, ot->ot_seq_rec_id, &ot->ot_seq_page, &ot->ot_seq_offset, ot->ot_thread))) + if (!(ot->ot_on_page = tab->tab_recs.xt_tc_get_page(ot->ot_rec_file, ot->ot_seq_rec_id, TAB_SEQ_LOAD_CACHE, &ot->ot_seq_page, &ot->ot_seq_offset, ot->ot_thread))) return FAILED; + if (!ot->ot_seq_page) { + XT_LOCK_MEMORY_PTR(ot->ot_seq_data, ot->ot_rec_file, xt_rec_id_to_rec_offset(tab, ot->ot_seq_rec_id), tab->tab_rows.tci_page_size, &ot->ot_thread->st_statistics.st_rec, ot->ot_thread); + if (!ot->ot_seq_data) + return FAILED; + ot->ot_on_page = TRUE; + ot->ot_seq_offset = 0; + } } next_record: @@ -5001,22 +4965,19 @@ xtPublic xtBool xt_tab_seq_next(XTOpenTa tab->tab_recs.xt_tc_release_page(ot->ot_rec_file, ot->ot_seq_page, ot->ot_thread); ot->ot_seq_page = NULL; } + if (ot->ot_seq_data) + /* NULL here means that in the case of non-memory mapped + * files we "keep" the lock. + */ + XT_UNLOCK_MEMORY_PTR(ot->ot_rec_file, ot->ot_seq_data, FALSE, ot->ot_thread); ot->ot_on_page = FALSE; goto next_page; } - if (ot->ot_seq_page) { - ptr_locked = FALSE; + if (ot->ot_seq_page) buff_ptr = ot->ot_seq_page->tcp_data + ot->ot_seq_offset; - } - else { - size_t red_size; - - ptr_locked = TRUE; - if (!xt_pread_fmap(ot->ot_rec_file, xt_rec_id_to_rec_offset(tab, ot->ot_seq_rec_id), sizeof(XTTabRecHeadDRec), sizeof(XTTabRecHeadDRec), &rec_head, &red_size, &ot->ot_thread->st_statistics.st_rec, ot->ot_thread)) - return FAILED; - buff_ptr = (xtWord1 *) &rec_head; - } + else + buff_ptr = ot->ot_seq_data + ot->ot_seq_offset; /* This is the current record: */ ot->ot_curr_rec_id = ot->ot_seq_rec_id; @@ -5033,7 +4994,6 @@ xtPublic xtBool xt_tab_seq_next(XTOpenTa case XT_ERR: goto failed; case XT_NEW: - ptr_locked = FALSE; buff_ptr = ot->ot_row_rbuffer; if (!xt_tab_get_rec_data(ot, new_rec_id, rec_size, ot->ot_row_rbuffer)) return XT_ERR; @@ -5066,8 +5026,6 @@ xtPublic xtBool xt_tab_seq_next(XTOpenTa invalid_rec = 0; goto next_record; default: - if (ptr_locked) - XT_LOCK_MEMORY_PTR(buff_ptr, ot->ot_rec_file, xt_rec_id_to_rec_offset(tab, ot->ot_curr_rec_id), tab->tab_rows.tci_page_size, &ot->ot_thread->st_statistics.st_rec, ot->ot_thread); break; } @@ -5099,17 +5057,176 @@ xtPublic xtBool xt_tab_seq_next(XTOpenTa break; } } - if (ptr_locked) - XT_UNLOCK_MEMORY_PTR(ot->ot_rec_file, ot->ot_thread); *eof = FALSE; return OK; failed_1: - if (ptr_locked) - XT_UNLOCK_MEMORY_PTR(ot->ot_rec_file, ot->ot_thread); failed: return FAILED; } +/* + * ----------------------------------------------------------------------- + * REPAIR TABLE + */ + +#define REP_FIND 0 +#define REP_ADD 1 +#define REP_DEL 2 + +static xtBool tab_exec_repair_pending(XTDatabaseHPtr db, int what, char *table_name) +{ + XTThreadPtr thread = xt_get_self(); + char file_path[PATH_MAX]; + XTOpenFilePtr of = NULL; + int len; + char *buffer = NULL, *ptr, *name; + char ch; + xtBool found = FALSE; + + xt_strcpy(PATH_MAX, file_path, db->db_main_path); + xt_add_pbxt_file(PATH_MAX, file_path, "repair-pending"); + + if (what == REP_ADD) { + if (!xt_open_file_ns(&of, file_path, XT_FS_CREATE | XT_FS_MAKE_PATH)) + return FALSE; + } + else { + if (!xt_open_file_ns(&of, file_path, XT_FS_DEFAULT)) + return FALSE; + } + if (!of) + return FALSE; + + len = (int) xt_seek_eof_file(NULL, of); + + if (!(buffer = (char *) xt_malloc_ns(len + 1))) + goto failed; + + if (!xt_pread_file(of, 0, len, len, buffer, NULL, &thread->st_statistics.st_x, thread)) + goto failed; + + buffer[len] = 0; + ptr = buffer; + for(;;) { + name = ptr; + while (*ptr && *ptr != '\n' && *ptr != '\r') + ptr++; + if (ptr > name) { + ch = *ptr; + *ptr = 0; + if (xt_tab_compare_names(name, table_name) == 0) { + *ptr = ch; + found = TRUE; + break; + } + *ptr = ch; + } + if (!*ptr) + break; + ptr++; + } + + switch (what) { + case REP_ADD: + if (!found) { + /* Remove any trailing empty lines: */ + while (len > 0) { + if (buffer[len-1] != '\n' && buffer[len-1] != '\r') + break; + len--; + } + if (len > 0) { + if (!xt_pwrite_file(of, len, 1, (void *) "\n", &thread->st_statistics.st_x, thread)) + goto failed; + len++; + } + if (!xt_pwrite_file(of, len, strlen(table_name), table_name, &thread->st_statistics.st_x, thread)) + goto failed; + len += strlen(table_name); + if (!xt_set_eof_file(NULL, of, len)) + goto failed; + } + break; + case REP_DEL: + if (found) { + if (*ptr != '\0') + ptr++; + memmove(name, ptr, len - (ptr - buffer)); + len = len - (ptr - name); + + /* Remove trailing empty lines: */ + while (len > 0) { + if (buffer[len-1] != '\n' && buffer[len-1] != '\r') + break; + len--; + } + + if (len > 0) { + if (!xt_pwrite_file(of, 0, len, buffer, &thread->st_statistics.st_x, thread)) + goto failed; + if (!xt_set_eof_file(NULL, of, len)) + goto failed; + } + } + break; + } + + xt_close_file_ns(of); + xt_free_ns(buffer); + + if (len == 0) + xt_fs_delete(NULL, file_path); + return found; + + failed: + if (of) + xt_close_file_ns(of); + if (buffer) + xt_free_ns(buffer); + xt_log_and_clear_exception(thread); + return FALSE; +} + +xtPublic void tab_make_table_name(XTTableHPtr tab, char *table_name, size_t size) +{ + char name_buf[XT_IDENTIFIER_NAME_SIZE*3+3]; + + xt_2nd_last_name_of_path(sizeof(name_buf), name_buf, tab->tab_name->ps_path); + myxt_static_convert_file_name(name_buf, table_name, size); + xt_strcat(size, table_name, "."); + myxt_static_convert_file_name(xt_last_name_of_path(tab->tab_name->ps_path), name_buf, sizeof(name_buf)); + xt_strcat(size, table_name, name_buf); +} + +xtPublic xtBool xt_tab_is_table_repair_pending(XTTableHPtr tab) +{ + char table_name[XT_IDENTIFIER_NAME_SIZE*3+3]; + + tab_make_table_name(tab, table_name, sizeof(table_name)); + return tab_exec_repair_pending(tab->tab_db, REP_FIND, table_name); +} + +xtPublic void xt_tab_table_repaired(XTTableHPtr tab) +{ + if (tab->tab_repair_pending) { + char table_name[XT_IDENTIFIER_NAME_SIZE*3+3]; + + tab->tab_repair_pending = FALSE; + tab_make_table_name(tab, table_name, sizeof(table_name)); + tab_exec_repair_pending(tab->tab_db, REP_DEL, table_name); + } +} + +xtPublic void xt_tab_set_table_repair_pending(XTTableHPtr tab) +{ + if (!tab->tab_repair_pending) { + char table_name[XT_IDENTIFIER_NAME_SIZE*3+3]; + + tab->tab_repair_pending = TRUE; + tab_make_table_name(tab, table_name, sizeof(table_name)); + tab_exec_repair_pending(tab->tab_db, REP_ADD, table_name); + } +} === modified file 'storage/pbxt/src/table_xt.h' --- a/storage/pbxt/src/table_xt.h 2009-04-02 10:03:14 +0000 +++ b/storage/pbxt/src/table_xt.h 2009-08-18 07:46:53 +0000 @@ -45,7 +45,17 @@ struct XTTablePath; #define XT_TAB_INCOMPATIBLE_VERSION 4 #define XT_TAB_CURRENT_VERSION 5 -#define XT_IND_CURRENT_VERSION 3 +/* This version of the index does not have lazy + * delete. The new version is compatible with + * this and maintains the old format. + */ +#define XT_IND_NO_LAZY_DELETE 3 +#define XT_IND_LAZY_DELETE_OK 4 +#ifdef XT_USE_LAZY_DELETE +#define XT_IND_CURRENT_VERSION XT_IND_LAZY_DELETE_OK +#else +#define XT_IND_CURRENT_VERSION XT_IND_NO_LAZY_DELETE +#endif #define XT_HEAD_BUFFER_SIZE 1024 @@ -100,15 +110,21 @@ struct XTTablePath; #define XT_TAB_POOL_CLOSED 3 /* Cannot open table at the moment, the pool is closed. */ #define XT_TAB_FAILED 4 -#define XT_TAB_ROW_USE_RW_MUTEX +#ifdef XT_NO_ATOMICS +#define XT_TAB_ROW_USE_PTHREAD_RW +#else +//#define XT_TAB_ROW_USE_RWMUTEX +//#define XT_TAB_ROW_USE_SPINXSLOCK +#define XT_TAB_ROW_USE_XSMUTEX +#endif -#ifdef XT_TAB_ROW_USE_FASTWRLOCK -#define XT_TAB_ROW_LOCK_TYPE XTFastRWLockRec -#define XT_TAB_ROW_INIT_LOCK(s, i) xt_fastrwlock_init(s, i) -#define XT_TAB_ROW_FREE_LOCK(s, i) xt_fastrwlock_free(s, i) -#define XT_TAB_ROW_READ_LOCK(i, s) xt_fastrwlock_slock(i, s) -#define XT_TAB_ROW_WRITE_LOCK(i, s) xt_fastrwlock_xlock(i, s) -#define XT_TAB_ROW_UNLOCK(i, s) xt_fastrwlock_unlock(i, s) +#ifdef XT_TAB_ROW_USE_XSMUTEX +#define XT_TAB_ROW_LOCK_TYPE XTXSMutexRec +#define XT_TAB_ROW_INIT_LOCK(s, i) xt_xsmutex_init_with_autoname(s, i) +#define XT_TAB_ROW_FREE_LOCK(s, i) xt_xsmutex_free(s, i) +#define XT_TAB_ROW_READ_LOCK(i, s) xt_xsmutex_slock(i, (s)->t_id) +#define XT_TAB_ROW_WRITE_LOCK(i, s) xt_xsmutex_xlock(i, (s)->t_id) +#define XT_TAB_ROW_UNLOCK(i, s) xt_xsmutex_unlock(i, (s)->t_id) #elif defined(XT_TAB_ROW_USE_PTHREAD_RW) #define XT_TAB_ROW_LOCK_TYPE xt_rwlock_type #define XT_TAB_ROW_INIT_LOCK(s, i) xt_init_rwlock(s, i) @@ -116,16 +132,23 @@ struct XTTablePath; #define XT_TAB_ROW_READ_LOCK(i, s) xt_slock_rwlock_ns(i) #define XT_TAB_ROW_WRITE_LOCK(i, s) xt_xlock_rwlock_ns(i) #define XT_TAB_ROW_UNLOCK(i, s) xt_unlock_rwlock_ns(i) -#elif defined(XT_TAB_ROW_USE_RW_MUTEX) +#elif defined(XT_TAB_ROW_USE_RWMUTEX) #define XT_TAB_ROW_LOCK_TYPE XTRWMutexRec #define XT_TAB_ROW_INIT_LOCK(s, i) xt_rwmutex_init_with_autoname(s, i) #define XT_TAB_ROW_FREE_LOCK(s, i) xt_rwmutex_free(s, i) #define XT_TAB_ROW_READ_LOCK(i, s) xt_rwmutex_slock(i, (s)->t_id) #define XT_TAB_ROW_WRITE_LOCK(i, s) xt_rwmutex_xlock(i, (s)->t_id) #define XT_TAB_ROW_UNLOCK(i, s) xt_rwmutex_unlock(i, (s)->t_id) +#elif defined(XT_TAB_ROW_USE_SPINXSLOCK) +#define XT_TAB_ROW_LOCK_TYPE XTSpinXSLockRec +#define XT_TAB_ROW_INIT_LOCK(s, i) xt_spinxslock_init_with_autoname(s, i) +#define XT_TAB_ROW_FREE_LOCK(s, i) xt_spinxslock_free(s, i) +#define XT_TAB_ROW_READ_LOCK(i, s) xt_spinxslock_slock(i, (s)->t_id) +#define XT_TAB_ROW_WRITE_LOCK(i, s) xt_spinxslock_xlock(i, (s)->t_id) +#define XT_TAB_ROW_UNLOCK(i, s) xt_spinxslock_unlock(i, (s)->t_id) #else #define XT_TAB_ROW_LOCK_TYPE XTSpinLockRec -#define XT_TAB_ROW_INIT_LOCK(s, i) xt_spinlock_init(s, i) +#define XT_TAB_ROW_INIT_LOCK(s, i) xt_spinlock_init_with_autoname(s, i) #define XT_TAB_ROW_FREE_LOCK(s, i) xt_spinlock_free(s, i) #define XT_TAB_ROW_READ_LOCK(i, s) xt_spinlock_lock(i) #define XT_TAB_ROW_WRITE_LOCK(i, s) xt_spinlock_lock(i) @@ -310,6 +333,7 @@ typedef struct XTTable : public XTHeap { /* Values that belong in the header when flushed! */ xtBool tab_flush_pending; /* TRUE if the table needs to be flushed */ xtBool tab_recovery_done; /* TRUE if the table has been recovered */ + xtBool tab_repair_pending; /* TRUE if the table has been marked for repair */ xtBool tab_temporary; /* TRUE if this is a temporary table {TEMP-TABLES}. */ off_t tab_bytes_to_flush; /* Number of bytes of the record/row files to flush. */ @@ -441,6 +465,9 @@ typedef struct XTOpenTable { xtRecordID ot_seq_rec_id; /* Current position of a sequential scan. */ xtRecordID ot_seq_eof_id; /* The EOF at the start of the sequential scan. */ XTTabCachePagePtr ot_seq_page; /* If ot_seq_buffer is non-NULL, then a page has been locked! */ + xtWord1 *ot_seq_data; /* Non-NULL if the data references memory mapped memory, or if it was + * allocated if no memory mapping is being used. + */ xtBool ot_on_page; size_t ot_seq_offset; /* Offset on the current page. */ } XTOpenTableRec, *XTOpenTablePtr; @@ -488,7 +515,7 @@ XTTableHPtr xt_use_table_no_lock(XTThr int xt_use_table_by_id(struct XTThread *self, XTTableHPtr *tab, struct XTDatabase *db, xtTableID tab_id); XTOpenTablePtr xt_open_table(XTTableHPtr tab); void xt_close_table(XTOpenTablePtr ot, xtBool flush, xtBool have_table_lock); -void xt_drop_table(struct XTThread *self, XTPathStrPtr name); +void xt_drop_table(struct XTThread *self, XTPathStrPtr name, xtBool drop_db); void xt_check_table(XTThreadPtr self, XTOpenTablePtr tab); void xt_rename_table(struct XTThread *self, XTPathStrPtr old_name, XTPathStrPtr new_name); @@ -536,8 +563,13 @@ xtBool xt_tab_put_eof_rec_data(XTOpen xtBool xt_tab_put_log_op_rec_data(XTOpenTablePtr ot, u_int status, xtRecordID free_rec_id, xtRecordID rec_id, size_t size, xtWord1 *buffer); xtBool xt_tab_put_log_rec_data(XTOpenTablePtr ot, u_int status, xtRecordID free_rec_id, xtRecordID rec_id, size_t size, xtWord1 *buffer, xtOpSeqNo *op_seq); xtBool xt_tab_get_rec_data(register XTOpenTablePtr ot, xtRecordID rec_id, size_t size, xtWord1 *buffer); +void xt_tab_disable_index(XTTableHPtr tab, u_int ind_error); void xt_tab_set_index_error(XTTableHPtr tab); +xtBool xt_tab_is_table_repair_pending(XTTableHPtr tab); +void xt_tab_table_repaired(XTTableHPtr tab); +void xt_tab_set_table_repair_pending(XTTableHPtr tab); + inline off_t xt_row_id_to_row_offset(register XTTableHPtr tab, xtRowID row_id) { return (off_t) tab->tab_rows.tci_header_size + (off_t) (row_id - 1) * (off_t) tab->tab_rows.tci_rec_size; @@ -605,3 +637,4 @@ inline xtIndexNodeID xt_ind_offset_to_no while (0) #endif + === modified file 'storage/pbxt/src/thread_xt.cc' --- a/storage/pbxt/src/thread_xt.cc 2009-04-02 10:03:14 +0000 +++ b/storage/pbxt/src/thread_xt.cc 2009-08-17 11:12:36 +0000 @@ -23,6 +23,10 @@ #include "xt_config.h" +#ifdef DRIZZLED +#include <bitset> +#endif + #ifndef XT_WIN #include <unistd.h> #include <sys/time.h> @@ -177,7 +181,7 @@ static void thr_log_newline(XTThreadPtr #endif #endif -void xt_log_flush(XTThreadPtr self __attribute__((unused))) +void xt_log_flush(XTThreadPtr XT_UNUSED(self)) { fflush(log_file); } @@ -466,7 +470,7 @@ static void thr_free_resources(XTThreadP } } -xtPublic void xt_bug(XTThreadPtr self __attribute__((unused))) +xtPublic void xt_bug(XTThreadPtr XT_UNUSED(self)) { static int *bug_ptr = NULL; @@ -532,7 +536,11 @@ xtPublic void xt_throw_error(XTThreadPtr #define XT_SYS_ERR_SIZE 300 -static c_char *thr_get_sys_error(int err, char *err_msg __attribute__((unused))) +#ifdef XT_WIN +static c_char *thr_get_sys_error(int err, char *err_msg) +#else +static c_char *thr_get_sys_error(int err, char *XT_UNUSED(err_msg)) +#endif { #ifdef XT_WIN char *ptr; @@ -638,7 +646,7 @@ static c_char *thr_get_err_string(int xt case XT_ERR_INDEX_CORRUPTED: str = "Table `%s` index is corrupted, REPAIR TABLE required"; break; case XT_ERR_NO_INDEX_CACHE: str = "Not enough index cache memory to handle concurrent updates"; break; case XT_ERR_INDEX_LOG_CORRUPT: str = "Index log corrupt: '%s'"; break; - case XT_ERR_TOO_MANY_THREADS: str = "Too many threads: %s, increase max_connections"; break; + case XT_ERR_TOO_MANY_THREADS: str = "Too many threads: %s, increase pbxt_max_threads"; break; case XT_ERR_TOO_MANY_WAITERS: str = "Too many waiting threads: %s"; break; case XT_ERR_INDEX_OLD_VERSION: str = "Table `%s` index created by an older version, REPAIR TABLE required"; break; case XT_ERR_PBXT_TABLE_EXISTS: str = "System table cannot be dropped because PBXT table still exists"; break; @@ -869,13 +877,18 @@ xtPublic void xt_log_errno(XTThreadPtr s * ----------------------------------------------------------------------- * Assertions and failures (one breakpoints for all failures) */ +//#define CRASH_ON_ASSERT -xtPublic xtBool xt_assert(XTThreadPtr self __attribute__((unused)), c_char *expr, c_char *func, c_char *file, u_int line) +xtPublic xtBool xt_assert(XTThreadPtr self, c_char *expr, c_char *func, c_char *file, u_int line) { + (void) self; #ifdef DEBUG //xt_set_fflush(TRUE); //xt_dump_trace(); printf("%s(%s:%d) %s\n", func, file, (int) line, expr); +#ifdef CRASH_ON_ASSERT + abort(); +#endif #ifdef XT_WIN FatalAppExit(0, "Assertion Failed!"); #endif @@ -981,11 +994,13 @@ static xtBool thr_setup_signals(void) } #endif -static void *thr_main(void *data) +typedef void *(*ThreadMainFunc)(XTThreadPtr self); + +extern "C" void *thr_main(void *data) { ThreadDataPtr td = (ThreadDataPtr) data; XTThreadPtr self = td->td_thr; - void *(*start_routine)(XTThreadPtr); + ThreadMainFunc start_routine; void *return_data; enter_(); @@ -1857,7 +1872,7 @@ xtPublic void xt_signal_thread(XTThreadP xt_broadcast_cond_ns(&target->t_cond); } -xtPublic void xt_terminate_thread(XTThreadPtr self __attribute__((unused)), XTThreadPtr target) +xtPublic void xt_terminate_thread(XTThreadPtr XT_UNUSED(self), XTThreadPtr target) { target->t_quit = TRUE; target->t_delayed_signal = SIGTERM; === modified file 'storage/pbxt/src/thread_xt.h' --- a/storage/pbxt/src/thread_xt.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/thread_xt.h 2009-08-17 11:12:36 +0000 @@ -44,14 +44,6 @@ * Macros and defines */ -#ifdef XT_WIN -#define __FUNC__ __FUNCTION__ -#elif defined(XT_SOLARIS) -#define __FUNC__ "__func__" -#else -#define __FUNC__ __PRETTY_FUNCTION__ -#endif - #define XT_ERR_MSG_SIZE (PATH_MAX + 200) #ifdef DEBUG @@ -291,6 +283,12 @@ typedef struct XTThread { xtBool st_xact_long_running; /* TRUE if this is a long running writer transaction. */ xtWord4 st_visible_time; /* Transactions committed before this time are visible. */ XTDataLogBufferRec st_dlog_buf; + + /* A list of the last 10 transactions run by this connection: */ +#ifdef XT_WAIT_FOR_CLEANUP + u_int st_last_xact; + xtXactID st_prev_xact[XT_MAX_XACT_BEHIND]; +#endif int st_xact_mode; /* The transaction mode. */ xtBool st_ignore_fkeys; /* TRUE if we must ignore foreign keys. */ === modified file 'storage/pbxt/src/trace_xt.cc' --- a/storage/pbxt/src/trace_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/trace_xt.cc 2009-08-17 11:12:36 +0000 @@ -72,6 +72,12 @@ xtPublic xtBool xt_init_trace(void) trace_log_offset = 0; trace_log_end = 0; trace_stat_count = 0; + +#ifdef XT_TRACK_CONNECTIONS + for (int i=0; i<XT_TRACK_MAX_CONNS; i++) + xt_track_conn_info[i].cu_t_id = i; +#endif + return TRUE; } @@ -343,3 +349,45 @@ xtPublic void xt_ftracef(char *fmt, ...) va_end(ap); } +/* + * ----------------------------------------------------------------------- + * CONNECTION TRACKING + */ + +#ifdef XT_TRACK_CONNECTIONS +XTConnInfoRec xt_track_conn_info[XT_TRACK_MAX_CONNS]; + +static int trace_comp_conn_info(const void *a, const void *b) +{ + XTConnInfoPtr ci_a = (XTConnInfoPtr) a, ci_b = (XTConnInfoPtr) b; + + if (ci_a->ci_curr_xact_id > ci_b->ci_curr_xact_id) + return 1; + if (ci_a->ci_curr_xact_id < ci_b->ci_curr_xact_id) + return -1; + return 0; +} + +xtPublic void xt_dump_conn_tracking(void) +{ + XTConnInfoRec conn_info[XT_TRACK_MAX_CONNS]; + XTConnInfoPtr ptr; + + memcpy(conn_info, xt_track_conn_info, sizeof(xt_track_conn_info)); + qsort(conn_info, XT_TRACK_MAX_CONNS, sizeof(XTConnInfoRec), trace_comp_conn_info); + + ptr = conn_info; + for (int i=0; i<XT_TRACK_MAX_CONNS; i++) { + if (ptr->ci_curr_xact_id || ptr->ci_prev_xact_id) { + printf("%3d curr=%d prev=%d prev-time=%ld\n", (int) ptr->cu_t_id, (int) ptr->ci_curr_xact_id, (int) ptr->ci_prev_xact_id, (long) ptr->ci_prev_xact_time); + if (i+1<XT_TRACK_MAX_CONNS) { + printf(" diff=%d\n", (int) (ptr+1)->ci_curr_xact_id - (int) ptr->ci_curr_xact_id); + } + } + ptr++; + } +} + +#endif + + === modified file 'storage/pbxt/src/trace_xt.h' --- a/storage/pbxt/src/trace_xt.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/trace_xt.h 2009-08-17 11:12:36 +0000 @@ -46,4 +46,29 @@ void xt_ftracef(char *fmt, ...); //#define PBXT_HANDLER_TRACE #endif +/* + * ----------------------------------------------------------------------- + * CONNECTION TRACKING + */ + +#define XT_TRACK_CONNECTIONS + +#ifdef XT_TRACK_CONNECTIONS +#define XT_TRACK_MAX_CONNS 500 + +typedef struct XTConnInfo { + xtThreadID cu_t_id; + xtXactID ci_curr_xact_id; + xtWord8 ci_xact_start; + + xtXactID ci_prev_xact_id; + xtWord8 ci_prev_xact_time; +} XTConnInfoRec, *XTConnInfoPtr; + +extern XTConnInfoRec xt_track_conn_info[XT_TRACK_MAX_CONNS]; + +void xt_dump_conn_tracking(void); + +#endif + #endif === modified file 'storage/pbxt/src/util_xt.cc' --- a/storage/pbxt/src/util_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/util_xt.cc 2009-08-17 11:12:36 +0000 @@ -61,7 +61,7 @@ xtPublic xtWord8 xt_time_now(void) return ms; } -xtPublic void xt_free_nothing(struct XTThread XT_UNUSED(*thr), void XT_UNUSED(*x)) +xtPublic void xt_free_nothing(struct XTThread *XT_UNUSED(thread), void *XT_UNUSED(x)) { } === modified file 'storage/pbxt/src/xaction_xt.cc' --- a/storage/pbxt/src/xaction_xt.cc 2009-04-02 10:03:14 +0000 +++ b/storage/pbxt/src/xaction_xt.cc 2009-08-17 11:12:36 +0000 @@ -23,6 +23,10 @@ #include "xt_config.h" +#ifdef DRIZZLED +#include <bitset> +#endif + #include <time.h> #include <signal.h> @@ -48,7 +52,7 @@ #endif static void xn_sw_wait_for_xact(XTThreadPtr self, XTDatabaseHPtr db, u_int hsecs); -static xtBool xn_get_xact_details(XTDatabaseHPtr db, xtXactID xn_id, XTThreadPtr thread __attribute__((unused)), int *flags, xtXactID *start, xtXactID *end, xtThreadID *thd_id); +static xtBool xn_get_xact_details(XTDatabaseHPtr db, xtXactID xn_id, XTThreadPtr XT_UNUSED(thread), int *flags, xtXactID *start, xtXactID *end, xtThreadID *thd_id); static xtBool xn_get_xact_pointer(XTDatabaseHPtr db, xtXactID xn_id, XTXactDataPtr *xact_ptr); /* ============================================================================================== */ @@ -203,7 +207,7 @@ typedef struct XNWaitFor { xtXactID wf_for_me_xn_id; /* The transaction we are waiting for. */ } XNWaitForRec, *XNWaitForPtr; -static int xn_compare_wait_for(XTThreadPtr XT_UNUSED(self), register const void XT_UNUSED(*thunk), register const void *a, register const void *b) +static int xn_compare_wait_for(XTThreadPtr XT_UNUSED(self), register const void *XT_UNUSED(thunk), register const void *a, register const void *b) { xtXactID *x = (xtXactID *) a; XNWaitForPtr y = (XNWaitForPtr) b; @@ -215,7 +219,7 @@ static int xn_compare_wait_for(XTThreadP return 1; } -static void xn_free_wait_for(XTThreadPtr XT_UNUSED(self), void XT_UNUSED(*thunk), void XT_UNUSED(*item)) +static void xn_free_wait_for(XTThreadPtr XT_UNUSED(self), void *XT_UNUSED(thunk), void *XT_UNUSED(item)) { } @@ -446,7 +450,9 @@ xtPublic xtBool xt_xn_wait_for_xact(XTTh xt_timed_wait_cond_ns(&my_wt->wt_cond, &my_wt->wt_lock, WAIT_FOR_XACT_TIME); } + /* Unreachable xt_unlock_mutex_ns(&my_wt->wt_lock); + */ } if (xw) { @@ -753,12 +759,13 @@ xtPublic xtXactID xt_xn_get_curr_id(XTDa return curr_xn_id; } -xtPublic XTXactDataPtr xt_xn_add_old_xact(XTDatabaseHPtr db, xtXactID xn_id, XTThreadPtr thread __attribute__((unused))) +xtPublic XTXactDataPtr xt_xn_add_old_xact(XTDatabaseHPtr db, xtXactID xn_id, XTThreadPtr thread) { register XTXactDataPtr xact; register XTXactSegPtr seg; register XTXactDataPtr *hash; + (void) thread; seg = &db->db_xn_idx[xn_id & XT_XN_SEGMENT_MASK]; XT_XACT_WRITE_LOCK(&seg->xs_tab_lock, thread); hash = &seg->xs_table[(xn_id >> XT_XN_SEGMENT_SHIFTS) % XT_XN_HASH_TABLE_SIZE]; @@ -778,7 +785,7 @@ xtPublic XTXactDataPtr xt_xn_add_old_xac */ db->db_sw_faster |= XT_SW_NO_MORE_XACT_SLOTS; if (!(xact = (XTXactDataPtr) xt_malloc_ns(sizeof(XTXactDataRec)))) { - XT_XACT_UNLOCK(&seg->xs_tab_lock, thread); + XT_XACT_UNLOCK(&seg->xs_tab_lock, thread, TRUE); return NULL; } } @@ -797,7 +804,7 @@ xtPublic XTXactDataPtr xt_xn_add_old_xac seg->xs_last_xn_id = xn_id; done_ok: - XT_XACT_UNLOCK(&seg->xs_tab_lock, thread); + XT_XACT_UNLOCK(&seg->xs_tab_lock, thread, TRUE); #ifdef HIGH_X tot_alloced++; if (tot_alloced > high_alloced) @@ -806,12 +813,13 @@ xtPublic XTXactDataPtr xt_xn_add_old_xac return xact; } -static XTXactDataPtr xn_add_new_xact(XTDatabaseHPtr db, xtXactID xn_id, XTThreadPtr thread __attribute__((unused))) +static XTXactDataPtr xn_add_new_xact(XTDatabaseHPtr db, xtXactID xn_id, XTThreadPtr thread) { register XTXactDataPtr xact; register XTXactSegPtr seg; register XTXactDataPtr *hash; + (void) thread; seg = &db->db_xn_idx[xn_id & XT_XN_SEGMENT_MASK]; XT_XACT_WRITE_LOCK(&seg->xs_tab_lock, thread); hash = &seg->xs_table[(xn_id >> XT_XN_SEGMENT_SHIFTS) % XT_XN_HASH_TABLE_SIZE]; @@ -825,7 +833,7 @@ static XTXactDataPtr xn_add_new_xact(XTD */ db->db_sw_faster |= XT_SW_NO_MORE_XACT_SLOTS; if (!(xact = (XTXactDataPtr) xt_malloc_ns(sizeof(XTXactDataRec)))) { - XT_XACT_UNLOCK(&seg->xs_tab_lock, thread); + XT_XACT_UNLOCK(&seg->xs_tab_lock, thread, TRUE); return NULL; } } @@ -841,7 +849,7 @@ static XTXactDataPtr xn_add_new_xact(XTD xact->xd_flags = 0; seg->xs_last_xn_id = xn_id; - XT_XACT_UNLOCK(&seg->xs_tab_lock, thread); + XT_XACT_UNLOCK(&seg->xs_tab_lock, thread, TRUE); #ifdef HIGH_X tot_alloced++; if (tot_alloced > high_alloced) @@ -850,7 +858,7 @@ static XTXactDataPtr xn_add_new_xact(XTD return xact; } -static xtBool xn_get_xact_details(XTDatabaseHPtr db, xtXactID xn_id, XTThreadPtr thread __attribute__((unused)), int *flags, xtXactID *start, xtWord4 *end, xtThreadID *thd_id) +static xtBool xn_get_xact_details(XTDatabaseHPtr db, xtXactID xn_id, XTThreadPtr XT_UNUSED(thread), int *flags, xtXactID *start, xtWord4 *end, xtThreadID *thd_id) { register XTXactSegPtr seg; register XTXactDataPtr xact; @@ -874,7 +882,7 @@ static xtBool xn_get_xact_details(XTData } xact = xact->xd_next_xact; } - XT_XACT_UNLOCK(&seg->xs_tab_lock, thread); + XT_XACT_UNLOCK(&seg->xs_tab_lock, thread, FALSE); return found; } @@ -900,11 +908,11 @@ static xtBool xn_get_xact_pointer(XTData } xact = xact->xd_next_xact; } - XT_XACT_UNLOCK(&seg->xs_tab_lock, thread); + XT_XACT_UNLOCK(&seg->xs_tab_lock, thread, FALSE); return found; } -static xtBool xn_get_xact_start(XTDatabaseHPtr db, xtXactID xn_id, XTThreadPtr thread __attribute__((unused)), xtLogID *log_id, xtLogOffset *log_offset) +static xtBool xn_get_xact_start(XTDatabaseHPtr db, xtXactID xn_id, XTThreadPtr XT_UNUSED(thread), xtLogID *log_id, xtLogOffset *log_offset) { register XTXactSegPtr seg; register XTXactDataPtr xact; @@ -922,12 +930,12 @@ static xtBool xn_get_xact_start(XTDataba } xact = xact->xd_next_xact; } - XT_XACT_UNLOCK(&seg->xs_tab_lock, thread); + XT_XACT_UNLOCK(&seg->xs_tab_lock, thread, FALSE); return found; } /* NOTE: this function may only be used by the sweeper or the recovery process. */ -xtPublic XTXactDataPtr xt_xn_get_xact(XTDatabaseHPtr db, xtXactID xn_id, XTThreadPtr thread __attribute__((unused))) +xtPublic XTXactDataPtr xt_xn_get_xact(XTDatabaseHPtr db, xtXactID xn_id, XTThreadPtr XT_UNUSED(thread)) { register XTXactSegPtr seg; register XTXactDataPtr xact; @@ -940,7 +948,7 @@ xtPublic XTXactDataPtr xt_xn_get_xact(XT break; xact = xact->xd_next_xact; } - XT_XACT_UNLOCK(&seg->xs_tab_lock, thread); + XT_XACT_UNLOCK(&seg->xs_tab_lock, thread, FALSE); return xact; } @@ -948,11 +956,12 @@ xtPublic XTXactDataPtr xt_xn_get_xact(XT * Delete a transaction, return TRUE if the transaction * was found. */ -xtPublic xtBool xt_xn_delete_xact(XTDatabaseHPtr db, xtXactID xn_id, XTThreadPtr thread __attribute__((unused))) +xtPublic xtBool xt_xn_delete_xact(XTDatabaseHPtr db, xtXactID xn_id, XTThreadPtr thread) { XTXactDataPtr xact, pxact = NULL; XTXactSegPtr seg; + (void) thread; seg = &db->db_xn_idx[xn_id & XT_XN_SEGMENT_MASK]; XT_XACT_WRITE_LOCK(&seg->xs_tab_lock, thread); xact = seg->xs_table[(xn_id >> XT_XN_SEGMENT_SHIFTS) % XT_XN_HASH_TABLE_SIZE]; @@ -963,13 +972,13 @@ xtPublic xtBool xt_xn_delete_xact(XTData else seg->xs_table[(xn_id >> XT_XN_SEGMENT_SHIFTS) % XT_XN_HASH_TABLE_SIZE] = xact->xd_next_xact; xn_free_xact(db, seg, xact); - XT_XACT_UNLOCK(&seg->xs_tab_lock, thread); + XT_XACT_UNLOCK(&seg->xs_tab_lock, thread, TRUE); return TRUE; } pxact = xact; xact = xact->xd_next_xact; } - XT_XACT_UNLOCK(&seg->xs_tab_lock, thread); + XT_XACT_UNLOCK(&seg->xs_tab_lock, thread, TRUE); return FALSE; } @@ -1253,6 +1262,10 @@ xtPublic xtBool xt_xn_begin(XTThreadPtr #ifdef TRACE_TRANSACTION xt_ttracef(self, "BEGIN T%lu\n", (u_long) self->st_xact_data->xd_start_xn_id); #endif +#ifdef XT_TRACK_CONNECTIONS + xt_track_conn_info[self->t_id].ci_curr_xact_id = self->st_xact_data->xd_start_xn_id; + xt_track_conn_info[self->t_id].ci_xact_start = xt_trace_clock(); +#endif return OK; } @@ -1375,6 +1388,13 @@ static xtBool xn_end_xact(XTThreadPtr th thread->st_xact_data = NULL; +#ifdef XT_TRACK_CONNECTIONS + xt_track_conn_info[thread->t_id].ci_prev_xact_id = xt_track_conn_info[thread->t_id].ci_curr_xact_id; + xt_track_conn_info[thread->t_id].ci_prev_xact_time = xt_trace_clock() - xt_track_conn_info[thread->t_id].ci_xact_start; + xt_track_conn_info[thread->t_id].ci_curr_xact_id = 0; + xt_track_conn_info[thread->t_id].ci_xact_start = 0; +#endif + xt_xn_wakeup_waiting_threads(thread); /* {WAKE-SW} Waking the sweeper @@ -1401,6 +1421,19 @@ static xtBool xn_end_xact(XTThreadPtr th /* Don't get too far ahead of the sweeper! */ if (writer) { +#ifdef XT_WAIT_FOR_CLEANUP + xtXactID wait_xn_id; + + /* This is the transaction that was committed 3 transactions ago: */ + wait_xn_id = thread->st_prev_xact[thread->st_last_xact]; + thread->st_prev_xact[thread->st_last_xact] = xn_id; + /* This works because XT_MAX_XACT_BEHIND == 2! */ + ASSERT_NS((thread->st_last_xact + 1) % XT_MAX_XACT_BEHIND == thread->st_last_xact ^ 1); + thread->st_last_xact ^= 1; + while (xt_xn_is_before(db->db_xn_to_clean_id, wait_xn_id) && (db->db_sw_faster & XT_SW_TOO_FAR_BEHIND)) { + xt_critical_wait(); + } +#else if ((db->db_sw_faster & XT_SW_TOO_FAR_BEHIND) != 0) { xtWord8 then = xt_trace_clock() + (xtWord8) 20000; @@ -1412,6 +1445,7 @@ static xtBool xn_end_xact(XTThreadPtr th break; } } +#endif } } return ok; @@ -1854,7 +1888,7 @@ static xtBool xn_sw_cleanup_done(XTThrea return FALSE; } -static void xn_sw_clean_indices(XTThreadPtr self __attribute__((unused)), XTOpenTablePtr ot, xtRecordID rec_id, xtRowID row_id, xtWord1 *rec_data, xtWord1 *rec_buffer) +static void xn_sw_clean_indices(XTThreadPtr XT_NDEBUG_UNUSED(self), XTOpenTablePtr ot, xtRecordID rec_id, xtRowID row_id, xtWord1 *rec_data, xtWord1 *rec_buffer) { XTTableHPtr tab = ot->ot_table; u_int cols_req; @@ -2599,7 +2633,13 @@ xtPublic void xt_wait_for_sweeper(XTThre if (db->db_sw_thread) { then = time(NULL); - while (!xt_xn_is_before(xt_xn_get_curr_id(db), db->db_xn_to_clean_id)) { // was db->db_xn_to_clean_id <= xt_xn_get_curr_id(db) + /* Changed xt_xn_get_curr_id(db) to db->db_xn_curr_id, + * This should work because we are not concerned about the difference + * between xt_xn_get_curr_id(db) and db->db_xn_curr_id, + * Which is just a matter of when transactions we can expect ot find + * in memory (see {GAP-INC-ADD-XACT}) + */ + while (!xt_xn_is_before(db->db_xn_curr_id, db->db_xn_to_clean_id)) { // was db->db_xn_to_clean_id <= xt_xn_get_curr_id(db) xt_lock_mutex(self, &db->db_sw_lock); pushr_(xt_unlock_mutex, &db->db_sw_lock); xt_wakeup_sweeper(db); === modified file 'storage/pbxt/src/xaction_xt.h' --- a/storage/pbxt/src/xaction_xt.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/xaction_xt.h 2009-08-17 11:12:36 +0000 @@ -36,28 +36,48 @@ struct XTOpenTable; #ifdef XT_USE_XACTION_DEBUG_SIZES -#define XT_XN_DATA_ALLOC_COUNT 400 -#define XT_XN_SEGMENT_SHIFTS 1 -#define XT_XN_HASH_TABLE_SIZE 31 #define XT_TN_NUMBER_INCREMENT 20 #define XT_TN_MAX_TO_FREE 20 #define XT_TN_MAX_TO_FREE_WASTE 3 #define XT_TN_MAX_TO_FREE_CHECK 3 #define XT_TN_MAX_TO_FREE_INC 3 +#define XT_XN_SEGMENT_SHIFTS 1 + #else -#define XT_XN_DATA_ALLOC_COUNT 1250 // Number of pre-allocated transaction data structures per segment -#define XT_XN_SEGMENT_SHIFTS 5 // (32) -#define XT_XN_HASH_TABLE_SIZE 1279 // This is a prime number! #define XT_TN_NUMBER_INCREMENT 100 // The increment of the transaction number on restart #define XT_TN_MAX_TO_FREE 800 // The maximum size of the "to free" list #define XT_TN_MAX_TO_FREE_WASTE 400 #define XT_TN_MAX_TO_FREE_CHECK 100 // Once we have exceeded the limit, we only try in intervals #define XT_TN_MAX_TO_FREE_INC 100 +//#define XT_XN_SEGMENT_SHIFTS 5 // (32) +//#define XT_XN_SEGMENT_SHIFTS 6 // (64) +//#define XT_XN_SEGMENT_SHIFTS 7 // (128) +#define XT_XN_SEGMENT_SHIFTS 8 // (256) +//#define XT_XN_SEGMENT_SHIFTS 9 // (512) + #endif +/* The hash table size (a prime number) */ +#if XT_XN_SEGMENT_SHIFTS == 1 // (1) +#define XT_XN_HASH_TABLE_SIZE 1301 +#elif XT_XN_SEGMENT_SHIFTS == 5 // (32) +#define XT_XN_HASH_TABLE_SIZE 1009 +#elif XT_XN_SEGMENT_SHIFTS == 6 // (64) +#define XT_XN_HASH_TABLE_SIZE 503 +#elif XT_XN_SEGMENT_SHIFTS == 7 // (128) +#define XT_XN_HASH_TABLE_SIZE 251 +#elif XT_XN_SEGMENT_SHIFTS == 8 // (256) +#define XT_XN_HASH_TABLE_SIZE 127 +#elif XT_XN_SEGMENT_SHIFTS == 9 // (512) +#define XT_XN_HASH_TABLE_SIZE 67 +#endif + +/* Number of pre-allocated transaction data structures per segment */ +#define XT_XN_DATA_ALLOC_COUNT XT_XN_HASH_TABLE_SIZE + #define XT_XN_NO_OF_SEGMENTS (1 << XT_XN_SEGMENT_SHIFTS) #define XT_XN_SEGMENT_MASK (XT_XN_NO_OF_SEGMENTS - 1) @@ -94,36 +114,34 @@ typedef struct XTXactData { } XTXactDataRec, *XTXactDataPtr; -#define XT_XACT_USE_SPINLOCK +#ifdef XT_NO_ATOMICS +#define XT_XACT_USE_PTHREAD_RW +#else +//#define XT_XACT_USE_SKEWRWLOCK +#define XT_XACT_USE_SPINXSLOCK +#endif -#ifdef XT_XACT_USE_FASTWRLOCK -#define XT_XACT_LOCK_TYPE XTFastRWLockRec -#define XT_XACT_INIT_LOCK(s, i) xt_fastrwlock_init(s, i) -#define XT_XACT_FREE_LOCK(s, i) xt_fastrwlock_free(s, i) -#define XT_XACT_READ_LOCK(i, s) xt_fastrwlock_slock(i, s) -#define XT_XACT_WRITE_LOCK(i, s) xt_fastrwlock_xlock(i, s) -#define XT_XACT_UNLOCK(i, s) xt_fastrwlock_unlock(i, s) -#elif defined(XT_XACT_USE_PTHREAD_RW) +#if defined(XT_XACT_USE_PTHREAD_RW) #define XT_XACT_LOCK_TYPE xt_rwlock_type #define XT_XACT_INIT_LOCK(s, i) xt_init_rwlock(s, i) #define XT_XACT_FREE_LOCK(s, i) xt_free_rwlock(i) #define XT_XACT_READ_LOCK(i, s) xt_slock_rwlock_ns(i) #define XT_XACT_WRITE_LOCK(i, s) xt_xlock_rwlock_ns(i) -#define XT_XACT_UNLOCK(i, s) xt_unlock_rwlock_ns(i) -#elif defined(XT_XACT_USE_RW_MUTEX) -#define XT_XACT_LOCK_TYPE XTRWMutexRec -#define XT_XACT_INIT_LOCK(s, i) xt_rwmutex_init(s, i) -#define XT_XACT_FREE_LOCK(s, i) xt_rwmutex_free(s, i) -#define XT_XACT_READ_LOCK(i, s) xt_rwmutex_slock(i, (s)->t_id) -#define XT_XACT_WRITE_LOCK(i, s) xt_rwmutex_xlock(i, (s)->t_id) -#define XT_XACT_UNLOCK(i, s) xt_rwmutex_unlock(i, (s)->t_id) +#define XT_XACT_UNLOCK(i, s, b) xt_unlock_rwlock_ns(i) +#elif defined(XT_XACT_USE_SPINXSLOCK) +#define XT_XACT_LOCK_TYPE XTSpinXSLockRec +#define XT_XACT_INIT_LOCK(s, i) xt_spinxslock_init_with_autoname(s, i) +#define XT_XACT_FREE_LOCK(s, i) xt_spinxslock_free(s, i) +#define XT_XACT_READ_LOCK(i, s) xt_spinxslock_slock(i) +#define XT_XACT_WRITE_LOCK(i, s) xt_spinxslock_xlock(i, (s)->t_id) +#define XT_XACT_UNLOCK(i, s, b) xt_spinxslock_unlock(i, b) #else -#define XT_XACT_LOCK_TYPE XTSpinLockRec -#define XT_XACT_INIT_LOCK(s, i) xt_spinlock_init_with_autoname(s, i) -#define XT_XACT_FREE_LOCK(s, i) xt_spinlock_free(s, i) -#define XT_XACT_READ_LOCK(i, s) xt_spinlock_lock(i) -#define XT_XACT_WRITE_LOCK(i, s) xt_spinlock_lock(i) -#define XT_XACT_UNLOCK(i, s) xt_spinlock_unlock(i) +#define XT_XACT_LOCK_TYPE XTSkewRWLockRec +#define XT_XACT_INIT_LOCK(s, i) xt_skewrwlock_init_with_autoname(s, i) +#define XT_XACT_FREE_LOCK(s, i) xt_skewrwlock_free(s, i) +#define XT_XACT_READ_LOCK(i, s) xt_skewrwlock_slock(i) +#define XT_XACT_WRITE_LOCK(i, s) xt_skewrwlock_xlock(i, (s)->t_id) +#define XT_XACT_UNLOCK(i, s, b) xt_skewrwlock_unlock(i, b) #endif /* We store the transactions in a number of segments, each === modified file 'storage/pbxt/src/xactlog_xt.cc' --- a/storage/pbxt/src/xactlog_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/xactlog_xt.cc 2009-08-18 07:46:53 +0000 @@ -28,6 +28,10 @@ #include "xt_config.h" +#ifdef DRIZZLED +#include <bitset> +#endif + #include <signal.h> #include "xactlog_xt.h" @@ -600,7 +604,12 @@ void XTDatabaseLog::xlog_setup(XTThreadP xt_init_mutex_with_autoname(self, &xl_write_lock); xt_init_cond(self, &xl_write_cond); +#ifdef XT_XLOG_WAIT_SPINS xt_writing = 0; + xt_waiting = 0; +#else + xt_writing = FALSE; +#endif xl_log_id = 0; xl_log_file = 0; @@ -752,6 +761,7 @@ xtBool XTDatabaseLog::xlog_append(XTThre xtLogOffset req_flush_log_offset; size_t part_size; xtWord8 flush_time; + xtWord2 sum; if (!size1) { /* Just flush the buffer... */ @@ -790,13 +800,13 @@ xtBool XTDatabaseLog::xlog_append(XTThre * enough space in the buffer, or a flush * is required. */ + xtWord8 then; /* * The objective of the following code is to * pick one writer, out of all threads. - * The others rest will wait for the writer. + * The rest will wait for the writer. */ - xtBool i_am_writer; if (write_reason == WR_FLUSH) { /* Before we flush, check if we should wait for running @@ -805,8 +815,7 @@ xtBool XTDatabaseLog::xlog_append(XTThre if (xl_db->db_xn_writer_count - xl_db->db_xn_writer_wait_count - xl_db->db_xn_long_running_count > 0 && xl_last_flush_time) { /* Wait for about as long as the last flush took, * the idea is to saturate the disk with flushing...: */ - xtWord8 then = xt_trace_clock() + (xtWord8) xl_last_flush_time; - + then = xt_trace_clock() + (xtWord8) xl_last_flush_time; for (;;) { xt_critical_wait(); /* If a thread leaves this loop because times up, or @@ -831,6 +840,55 @@ xtBool XTDatabaseLog::xlog_append(XTThre } } +#ifdef XT_XLOG_WAIT_SPINS + /* Spin for 1/1000s: */ + then = xt_trace_clock() + (xtWord8) 1000; + for (;;) { + if (!xt_atomic_tas4(&xt_writing, 1)) + break; + + /* If I am not the writer, then I just waited for the + * writer. So it may be that my requirements have now + * been met! + */ + if (write_reason == WR_FLUSH) { + /* If the reason was to flush, then + * check the last flush sequence, maybe it is passed + * our required sequence. + */ + if (xt_comp_log_pos(req_flush_log_id, req_flush_log_offset, xl_flush_log_id, xl_flush_log_offset) <= 0) { + /* The required flush position of the log is before + * or equal to the actual flush position. This means the condition + * for this thread have been satified (via group commit). + * Nothing more to do! + */ + ASSERT_NS(xt_comp_log_pos(xl_write_log_id, xl_write_log_offset, xl_append_log_id, xl_append_log_offset) <= 0); + return OK; + } + } + else { + /* It may be that there is now space in the append buffer: */ + if (xl_append_buf_pos + size1 + size2 <= xl_size_of_buffers) + goto copy_to_log_buffer; + } + + if (xt_trace_clock() >= then) { + xt_lock_mutex_ns(&xl_write_lock); + xt_waiting++; + if (!xt_timed_wait_cond_ns(&xl_write_cond, &xl_write_lock, 500)) { + xt_waiting--; + xt_unlock_mutex_ns(&xl_write_lock); + return FALSE; + } + xt_waiting--; + xt_unlock_mutex_ns(&xl_write_lock); + } + else + xt_critical_wait(); + } +#else + xtBool i_am_writer; + i_am_writer = FALSE; xt_lock_mutex_ns(&xl_write_lock); if (xt_writing) { @@ -873,6 +931,7 @@ xtBool XTDatabaseLog::xlog_append(XTThre goto write_log_to_file; } +#endif /* I am the writer, check the conditions, again: */ if (write_reason == WR_FLUSH) { @@ -881,8 +940,14 @@ xtBool XTDatabaseLog::xlog_append(XTThre /* The writers required flush position is before or equal * to the actual position, so the writer is done... */ +#ifdef XT_XLOG_WAIT_SPINS + xt_writing = 0; + if (xt_waiting) + xt_cond_wakeall(&xl_write_cond); +#else xt_writing = FALSE; xt_cond_wakeall(&xl_write_cond); +#endif ASSERT_NS(xt_comp_log_pos(xl_write_log_id, xl_write_log_offset, xl_append_log_id, xl_append_log_offset) <= 0); return OK; } @@ -923,8 +988,14 @@ xtBool XTDatabaseLog::xlog_append(XTThre xt_unlock_mutex_ns(&xl_db->db_wr_lock); } } +#ifdef XT_XLOG_WAIT_SPINS + xt_writing = 0; + if (xt_waiting) + xt_cond_wakeall(&xl_write_cond); +#else xt_writing = FALSE; xt_cond_wakeall(&xl_write_cond); +#endif ASSERT_NS(xt_comp_log_pos(xl_write_log_id, xl_write_log_offset, xl_append_log_id, xl_append_log_offset) <= 0); return ok; } @@ -934,8 +1005,14 @@ xtBool XTDatabaseLog::xlog_append(XTThre * to copy our data into the buffer: */ if (xl_append_buf_pos + size1 + size2 <= xl_size_of_buffers) { +#ifdef XT_XLOG_WAIT_SPINS + xt_writing = 0; + if (xt_waiting) + xt_cond_wakeall(&xl_write_cond); +#else xt_writing = FALSE; xt_cond_wakeall(&xl_write_cond); +#endif goto copy_to_log_buffer; } } @@ -1055,6 +1132,7 @@ xtBool XTDatabaseLog::xlog_append(XTThre /* [(8)] Flush the compactor log. */ xt_lock_mutex_ns(&xl_db->db_co_dlog_lock); if (!xl_db->db_co_thread->st_dlog_buf.dlb_flush_log(TRUE, thread)) { + xl_log_bytes_written -= part_size; xt_unlock_mutex_ns(&xl_db->db_co_dlog_lock); goto write_failed; } @@ -1063,8 +1141,10 @@ xtBool XTDatabaseLog::xlog_append(XTThre /* And flush if required: */ flush_time = thread->st_statistics.st_xlog.ts_flush_time; - if (!xt_flush_file(xl_log_file, &thread->st_statistics.st_xlog, thread)) + if (!xt_flush_file(xl_log_file, &thread->st_statistics.st_xlog, thread)) { + xl_log_bytes_written -= part_size; goto write_failed; + } xl_last_flush_time = (u_int) (thread->st_statistics.st_xlog.ts_flush_time - flush_time); xl_log_bytes_flushed = xl_log_bytes_written; @@ -1085,8 +1165,14 @@ xtBool XTDatabaseLog::xlog_append(XTThre * position, continue writing: */ goto rewrite; +#ifdef XT_XLOG_WAIT_SPINS + xt_writing = 0; + if (xt_waiting) + xt_cond_wakeall(&xl_write_cond); +#else xt_writing = FALSE; xt_cond_wakeall(&xl_write_cond); +#endif ASSERT_NS(xt_comp_log_pos(xl_write_log_id, xl_write_log_offset, xl_append_log_id, xl_append_log_offset) <= 0); return OK; } @@ -1100,8 +1186,14 @@ xtBool XTDatabaseLog::xlog_append(XTThre if (xl_append_buf_pos + size1 + size2 > xl_size_of_buffers) goto rewrite; +#ifdef XT_XLOG_WAIT_SPINS + xt_writing = 0; + if (xt_waiting) + xt_cond_wakeall(&xl_write_cond); +#else xt_writing = FALSE; xt_cond_wakeall(&xl_write_cond); +#endif } copy_to_log_buffer: @@ -1146,8 +1238,6 @@ xtBool XTDatabaseLog::xlog_append(XTThre case XT_LOG_ENT_DELETE_BG: case XT_LOG_ENT_DELETE_FL: case XT_LOG_ENT_DELETE_FL_BG: - xtWord2 sum; - sum = XT_GET_DISK_2(record->xu.xu_checksum_2) ^ XT_CHECKSUM_2(xl_append_log_id); XT_SET_DISK_2(record->xu.xu_checksum_2, sum); @@ -1158,6 +1248,10 @@ xtBool XTDatabaseLog::xlog_append(XTThre xl_db->db_xn_total_writer_count++; } break; + case XT_LOG_ENT_REC_REMOVED_BI: + sum = XT_GET_DISK_2(record->xu.xu_checksum_2) ^ XT_CHECKSUM_2(xl_append_log_id); + XT_SET_DISK_2(record->xu.xu_checksum_2, sum); + break; case XT_LOG_ENT_ROW_NEW: case XT_LOG_ENT_ROW_NEW_FL: record->xl.xl_checksum_1 ^= XT_CHECKSUM_1(xl_append_log_id); @@ -1209,8 +1303,14 @@ xtBool XTDatabaseLog::xlog_append(XTThre return OK; write_failed: +#ifdef XT_XLOG_WAIT_SPINS + xt_writing = 0; + if (xt_waiting) + xt_cond_wakeall(&xl_write_cond); +#else xt_writing = FALSE; xt_cond_wakeall(&xl_write_cond); +#endif return FAILED; } @@ -1595,7 +1695,7 @@ void XTDatabaseLog::xlog_seq_close(XTXac seq->xseq_log_eof = 0; } -xtBool XTDatabaseLog::xlog_seq_start(XTXactSeqReadPtr seq, xtLogID log_id, xtLogOffset log_offset, xtBool missing_ok __attribute__((unused))) +xtBool XTDatabaseLog::xlog_seq_start(XTXactSeqReadPtr seq, xtLogID log_id, xtLogOffset log_offset, xtBool XT_UNUSED(missing_ok)) { if (seq->xseq_rec_log_id != log_id) { seq->xseq_rec_log_id = log_id; @@ -2094,7 +2194,9 @@ xtBool XTDatabaseLog::xlog_seq_next(XTXa goto return_empty; } default: - ASSERT_NS(FALSE); + /* It is possible to land here after a crash, if the + * log was not completely written. + */ seq->xseq_record_len = 0; goto return_empty; } @@ -2304,7 +2406,13 @@ static void xlog_wr_wait_for_log_flush(X * the wait, and the sweeper has nothing to do, and the checkpointer. */ if (db->db_xn_curr_id == last_xn_id && - xt_xn_is_before(xt_xn_get_curr_id(db), db->db_xn_to_clean_id) && // db->db_xn_curr_id < db->db_xn_to_clean_id + /* Changed xt_xn_get_curr_id(db) to db->db_xn_curr_id, + * This should work because we are not concerned about the difference + * between xt_xn_get_curr_id(db) and db->db_xn_curr_id, + * Which is just a matter of when transactions we can expect ot find + * in memory (see {GAP-INC-ADD-XACT}) + */ + xt_xn_is_before(db->db_xn_curr_id, db->db_xn_to_clean_id) && // db->db_xn_curr_id < db->db_xn_to_clean_id !db->db_restart.xres_is_checkpoint_pending(db->db_xlog.xl_write_log_id, db->db_xlog.xl_write_log_offset)) { /* There seems to be no activity at the moment. * this might be a good time to write the log data. @@ -2409,9 +2517,6 @@ static void xlog_wr_main(XTThreadPtr sel if (!record) { break; } - /* Count the number of bytes read from the log: */ - db->db_xlog.xl_log_bytes_read += ws->ws_seqread.xseq_record_len; - switch (record->xl.xl_status_1) { case XT_LOG_ENT_HEADER: break; @@ -2435,6 +2540,8 @@ static void xlog_wr_main(XTThreadPtr sel xt_xres_apply_in_order(self, ws, ws->ws_seqread.xseq_rec_log_id, ws->ws_seqread.xseq_rec_log_offset, record); break; } + /* Count the number of bytes read from the log: */ + db->db_xlog.xl_log_bytes_read += ws->ws_seqread.xseq_record_len; } } === modified file 'storage/pbxt/src/xactlog_xt.h' --- a/storage/pbxt/src/xactlog_xt.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/xactlog_xt.h 2009-08-17 11:12:36 +0000 @@ -373,6 +373,13 @@ typedef struct XTXactLogFile { /* * The transaction log. Each database has one. */ + +/* Does not seem to make much difference... */ +#ifndef XT_NO_ATOMICS +/* This function uses atomic ops: */ +//#define XT_XLOG_WAIT_SPINS +#endif + typedef struct XTDatabaseLog { struct XTDatabase *xl_db; @@ -390,7 +397,12 @@ typedef struct XTDatabaseLog { /* The writer log buffer: */ xt_mutex_type xl_write_lock; xt_cond_type xl_write_cond; +#ifdef XT_XLOG_WAIT_SPINS + xtWord4 xt_writing; /* 1 if a thread is writing. */ + xtWord4 xt_waiting; /* Count of the threads waiting on the xl_write_cond. */ +#else xtBool xt_writing; /* TRUE if a thread is writing. */ +#endif xtLogID xl_log_id; /* The number of the write log. */ XTOpenFilePtr xl_log_file; /* The open write log. */ === modified file 'storage/pbxt/src/xt_config.h' --- a/storage/pbxt/src/xt_config.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/xt_config.h 2009-08-31 11:07:44 +0000 @@ -81,7 +81,8 @@ const int max_connections = 500; #define DEBUG #endif // _DEBUG #else -#define XT_STREAMING +// Paul suggested to disable PBMS in MariaDB for now. +// #define PBMS_ENABLED #endif #ifdef __FreeBSD__ @@ -96,4 +97,22 @@ const int max_connections = 500; #define XT_SOLARIS #endif +/* + * Definition of which atomic operations to use: + */ +#ifdef XT_WIN +/* MS Studio style embedded assembler for x86 */ +#define XT_ATOMIC_WIN32_X86 +#elif defined(__GNUC__) && (defined(__x86_64__) || defined(__i386__)) +/* Use GNU style embedded assembler for x86 */ +#define XT_ATOMIC_GNUC_X86 +#elif defined(XT_SOLARIS) +/* Use Sun atomic operations library + * http://docs.sun.com/app/docs/doc/816-5168/atomic-ops-3c?a=view + */ +#define XT_ATOMIC_SOLARIS_LIB +#else +#define XT_NO_ATOMICS +#endif + #endif === modified file 'storage/pbxt/src/xt_defs.h' --- a/storage/pbxt/src/xt_defs.h 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/xt_defs.h 2009-08-17 11:12:36 +0000 @@ -187,7 +187,16 @@ typedef struct XTPathStr { char ps_path[XT_VAR_LENGTH]; } *XTPathStrPtr; -#define XT_UNUSED(x) x __attribute__((__unused__)) +//#define XT_UNUSED(x) x __attribute__((__unused__)) +#define XT_UNUSED(x) + +/* Only used when DEBUG is on: */ +#ifdef DEBUG +#define XT_NDEBUG_UNUSED(x) x +#else +//#define XT_NDEBUG_UNUSED(x) x __attribute__((__unused__)) +#define XT_NDEBUG_UNUSED(x) +#endif /* ---------------------------------------------------------------------- * MAIN CONSTANTS @@ -267,8 +276,10 @@ typedef struct XTPathStr { * the row list is scanned. * * For more details see [(9)]. + * 223, 1019, 3613 */ -#define XT_ROW_RWLOCKS 223 +#define XT_ROW_RWLOCKS 1019 +//#define XT_ROW_RWLOCKS 223 /* * These are the number of row lock "slots" per table. @@ -292,31 +303,20 @@ typedef struct XTPathStr { */ #define XT_OPEN_TABLE_FREE_TIME 30 -#ifdef XT_USE_GLOBAL_DEBUG_SIZES -/* - * DEBUG SIZES! - * Reduce the thresholds to make things happen faster. +/* Define this in order to use memory mapped files + * (record and row pointer files only). + * + * This makes no difference in sysbench R/W performance + * test. */ +//#define XT_USE_ROW_REC_MMAP_FILES -//#undef XT_ROW_RWLOCKS -//#define XT_ROW_RWLOCKS 2 - -//#undef XT_TAB_MIN_VAR_REC_LENGTH -//#define XT_TAB_MIN_VAR_REC_LENGTH 20 - -//#undef XT_ROW_LOCK_COUNT -//#define XT_ROW_LOCK_COUNT (XT_ROW_RWLOCKS * 2) - -//#undef XT_INDEX_PAGE_SHIFTS -//#define XT_INDEX_PAGE_SHIFTS 12 - -//#undef XT_INDEX_WRITE_BUFFER_SIZE -//#define XT_INDEX_WRITE_BUFFER_SIZE (40 * 1024) - -#endif - -/* Define this in order to use memory mapped files: */ -#define XT_USE_ROW_REC_MMAP_FILES +/* Define this if sequential scan should load data into the + * record cache. + * + * This is the way InnoDB behaves. + */ +#define XT_SEQ_SCAN_LOADS_CACHE /* Define this in order to use direct I/O on index files: */ /* NOTE: DO NOT ENABLE! @@ -326,32 +326,34 @@ typedef struct XTPathStr { */ //#define XT_USE_DIRECT_IO_ON_INDEX -#ifdef XT_USE_ROW_REC_MMAP_FILES - -#define XT_SEQ_SCAN_FROM_MEMORY -#define XT_ROW_REC_FILE_PTR XTMapFilePtr -#define XT_PWRITE_RR_FILE xt_pwrite_fmap -#define XT_PREAD_RR_FILE xt_pread_fmap -#define XT_FLUSH_RR_FILE xt_flush_fmap -#define XT_CLOSE_RR_FILE_NS xt_close_fmap_ns - -#else - -#define XT_ROW_REC_FILE_PTR XTOpenFilePtr -#define XT_PWRITE_RR_FILE xt_pwrite_file -#define XT_PREAD_RR_FILE xt_pread_file -#define XT_FLUSH_RR_FILE xt_flush_file -#define XT_CLOSE_RR_FILE_NS xt_close_file_ns +/* + * Define this variable if PBXT should do lazy deleting in indexes + * Note, even if the variable is not defined, PBXT will handle + * lazy deleted items in an index. + * + * NOTE: This can cause significant degrade of index scan speed. + * 25% on sysbench readonly index scan tests. + */ +//#define XT_USE_LAZY_DELETE -#endif +/* + * Define this variable if a connection should wait for the + * sweeper to clean up previous transactions executed by the + * connection, before continuing. + * + * The number of transactions that the sweeper is aload to + * lag can be dynamic, but there is a limit (XT_MAX_XACT_BEHIND) + */ +#define XT_WAIT_FOR_CLEANUP -#ifdef XT_SEQ_SCAN_FROM_MEMORY -#define XT_LOCK_MEMORY_PTR(x, f, a, s, v, c) do { x = xt_lock_fmap_ptr(f, a, s, v, c); } while (0) -#define XT_UNLOCK_MEMORY_PTR(f, v) xt_unlock_fmap_ptr(f, v); -#else -#define XT_LOCK_MEMORY_PTR(x, f, a, v, c) -#define XT_UNLOCK_MEMORY_PTR(f, v) -#endif +/* + * This seems to be the optimal value, at least according to + * sysbench/sysbench run --test=oltp --num-threads=128 --max-requests=50000 --mysql-user=root + * --oltp-table-size=100000 --oltp-table-name=sb_pbxt --mysql-engine-trx=yes + * + * Using 8, 16 and 128 threads. + */ +#define XT_MAX_XACT_BEHIND 2 /* {NO-ACTION-BUG} * Define this to implement NO ACTION correctly @@ -405,6 +407,60 @@ typedef struct XTPathStr { #define XT_ADD_PTR(p, l) ((void *) ((char *) (p) + (l))) /* ---------------------------------------------------------------------- + * DEFINES DEPENDENT ON CONSTANTS + */ + +#ifdef XT_USE_ROW_REC_MMAP_FILES + +#define XT_ROW_REC_FILE_PTR XTMapFilePtr +#define XT_PWRITE_RR_FILE xt_pwrite_fmap +#define XT_PREAD_RR_FILE xt_pread_fmap +#define XT_FLUSH_RR_FILE xt_flush_fmap +#define XT_CLOSE_RR_FILE_NS xt_close_fmap_ns + +#define XT_LOCK_MEMORY_PTR(x, f, a, s, v, c) do { x = xt_lock_fmap_ptr(f, a, s, v, c); } while (0) +#define XT_UNLOCK_MEMORY_PTR(f, d, e, v) do { xt_unlock_fmap_ptr(f, v); d = NULL; } while (0) + +#else + +#define XT_ROW_REC_FILE_PTR XTOpenFilePtr +#define XT_PWRITE_RR_FILE xt_pwrite_file +#define XT_PREAD_RR_FILE xt_pread_file +#define XT_FLUSH_RR_FILE xt_flush_file +#define XT_CLOSE_RR_FILE_NS xt_close_file_ns + +#define XT_LOCK_MEMORY_PTR(x, f, a, s, v, c) do { if (!xt_lock_file_ptr(f, &x, a, s, v, c)) x = NULL; } while (0) +#define XT_UNLOCK_MEMORY_PTR(f, d, e, v) do { if (e) { xt_unlock_file_ptr(f, d, v); d = NULL; } } while (0) + +#endif + +/* ---------------------------------------------------------------------- + * DEBUG SIZES! + * Reduce the thresholds to make things happen faster. + */ + +#ifdef XT_USE_GLOBAL_DEBUG_SIZES + +//#undef XT_ROW_RWLOCKS +//#define XT_ROW_RWLOCKS 2 + +//#undef XT_TAB_MIN_VAR_REC_LENGTH +//#define XT_TAB_MIN_VAR_REC_LENGTH 20 + +//#undef XT_ROW_LOCK_COUNT +//#define XT_ROW_LOCK_COUNT (XT_ROW_RWLOCKS * 2) + +//#undef XT_INDEX_PAGE_SHIFTS +//#define XT_INDEX_PAGE_SHIFTS 8 // 256 +//#undef XT_BLOCK_SIZE_FOR_DIRECT_IO +//#define XT_BLOCK_SIZE_FOR_DIRECT_IO 256 + +//#undef XT_INDEX_WRITE_BUFFER_SIZE +//#define XT_INDEX_WRITE_BUFFER_SIZE (40 * 1024) + +#endif + +/* ---------------------------------------------------------------------- * BYTE ORDER */ @@ -645,6 +701,14 @@ typedef struct xtIndexNodeID { #define XT_XACT_ID_SIZE 4 #define XT_CHECKSUM4_XACT(x) (x) +#ifdef XT_WIN +#define __FUNC__ __FUNCTION__ +#elif defined(XT_SOLARIS) +#define __FUNC__ "__func__" +#else +#define __FUNC__ __PRETTY_FUNCTION__ +#endif + /* ---------------------------------------------------------------------- * GLOBAL VARIABLES */ @@ -669,6 +733,7 @@ extern xtBool pbxt_crash_debug; #define MYSQL_THD Session * #define THR_THD THR_Session #define STRUCT_TABLE class Table +#define TABLE_SHARE TableShare #define MYSQL_TYPE_STRING DRIZZLE_TYPE_VARCHAR #define MYSQL_TYPE_VARCHAR DRIZZLE_TYPE_VARCHAR @@ -687,6 +752,7 @@ extern xtBool pbxt_crash_debug; #define mx_tmp_use_all_columns(x, y) (x)->use_all_columns(y) #define mx_tmp_restore_column_map(x, y) (x)->restore_column_map(y) +#define MX_BIT_FAST_TEST_AND_SET(x, y) bitmap_test_and_set(x, y) #define MX_TABLE_TYPES_T handler::Table_flags #define MX_UINT8_T uint8_t @@ -696,6 +762,7 @@ extern xtBool pbxt_crash_debug; #define MX_CHARSET_INFO struct charset_info_st #define MX_CONST_CHARSET_INFO const struct charset_info_st #define MX_CONST const + #define my_bool bool #define int16 int16_t #define int32 int32_t @@ -712,6 +779,9 @@ extern xtBool pbxt_crash_debug; #define HA_CAN_SQL_HANDLER 0 #define HA_CAN_INSERT_DELAYED 0 +#define HA_BINLOG_ROW_CAPABLE 0 +#define HA_BINLOG_STMT_CAPABLE 0 +#define HA_CACHE_TBL_TRANSACT 0 #define max cmax #define min cmin @@ -734,6 +804,7 @@ extern xtBool pbxt_crash_debug; #define thd_tablespace_op session_tablespace_op #define thd_alloc session_alloc #define thd_make_lex_string session_make_lex_string +#define column_bitmaps_signal() #define my_pthread_setspecific_ptr(T, V) pthread_setspecific(T, (void*) (V)) @@ -750,6 +821,9 @@ extern xtBool pbxt_crash_debug; (((uint32_t) (((const unsigned char*) (A))[1])) << 16) +\ (((uint32_t) (((const unsigned char*) (A))[0])) << 24))) +class PBXTStorageEngine; +typedef PBXTStorageEngine handlerton; + #else // DRIZZLED /* The MySQL case: */ #if MYSQL_VERSION_ID >= 60008 @@ -760,6 +834,7 @@ extern xtBool pbxt_crash_debug; #define mx_tmp_use_all_columns dbug_tmp_use_all_columns #define mx_tmp_restore_column_map(x, y) dbug_tmp_restore_column_map((x)->read_set, y) +#define MX_BIT_FAST_TEST_AND_SET(x, y) bitmap_fast_test_and_set(x, y) #define MX_TABLE_TYPES_T ulonglong #define MX_UINT8_T uint8 @@ -772,6 +847,11 @@ extern xtBool pbxt_crash_debug; #endif // DRIZZLED +#define MX_BITMAP MY_BITMAP +#define MX_BIT_SIZE() n_bits +#define MX_BIT_IS_SUBSET(x, y) bitmap_is_subset(x, y) +#define MX_BIT_SET(x, y) bitmap_set_bit(x, y) + #ifndef XT_SCAN_CORE_DEFINED #define XT_SCAN_CORE_DEFINED xtBool xt_mm_scan_core(void);

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2723)
by knielsen＠knielsen-hq.org 31 Aug '09

31 Aug '09

#At lp:maria 2723 knielsen(a)knielsen-hq.org 2009-08-31 PBXT merge fixes. - Disable PBMS in MariaDB (as per recommendation of Paul). - Add missing *-master.opt files to make testsuite work on case-sensitive file systems. - Result file updates. added: mysql-test/suite/pbxt/t/lowercase_table_grant-master.opt mysql-test/suite/pbxt/t/lowercase_table_qcache-master.opt mysql-test/suite/pbxt/t/lowercase_view-master.opt mysql-test/suite/pbxt/t/udf-master.opt modified: mysql-test/suite/pbxt/r/lowercase_view.result mysql-test/suite/pbxt/r/ps_1general.result storage/pbxt/src/Makefile.am storage/pbxt/src/xt_config.h === modified file 'mysql-test/suite/pbxt/r/lowercase_view.result' --- a/mysql-test/suite/pbxt/r/lowercase_view.result 2009-04-02 20:36:52 +0000 +++ b/mysql-test/suite/pbxt/r/lowercase_view.result 2009-08-31 11:07:44 +0000 @@ -119,7 +119,7 @@ create table t1Aa (col1 int); create view v1Aa as select col1 from t1Aa as AaA; show create view v1AA; View Create View character_set_client collation_connection -v1aa CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1aa` AS select `AaA`.`col1` AS `col1` from `t1aa` `AaA` latin1 latin1_swedish_ci +v1aa CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1aa` AS select `aaa`.`col1` AS `col1` from `t1aa` `aaa` latin1 latin1_swedish_ci drop view v1AA; select Aaa.col1 from t1Aa as AaA; col1 @@ -128,7 +128,7 @@ drop view v1AA; create view v1Aa as select AaA.col1 from t1Aa as AaA; show create view v1AA; View Create View character_set_client collation_connection -v1aa CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1aa` AS select `AaA`.`col1` AS `col1` from `t1aa` `AaA` latin1 latin1_swedish_ci +v1aa CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1aa` AS select `aaa`.`col1` AS `col1` from `t1aa` `aaa` latin1 latin1_swedish_ci drop view v1AA; drop table t1Aa; CREATE TABLE t1 (a int, b int); @@ -142,7 +142,7 @@ CREATE OR REPLACE VIEW v1 AS select X.a from t1 AS X group by X.b having (X.a = 1); SHOW CREATE VIEW v1; View Create View character_set_client collation_connection -v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select `X`.`a` AS `a` from `t1` `X` group by `X`.`b` having (`X`.`a` = 1) latin1 latin1_swedish_ci +v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select `x`.`a` AS `a` from `t1` `x` group by `x`.`b` having (`x`.`a` = 1) latin1 latin1_swedish_ci SELECT * FROM v1; a DROP VIEW v1; === modified file 'mysql-test/suite/pbxt/r/ps_1general.result' --- a/mysql-test/suite/pbxt/r/ps_1general.result 2009-08-17 15:57:58 +0000 +++ b/mysql-test/suite/pbxt/r/ps_1general.result 2009-08-31 11:07:44 +0000 @@ -293,7 +293,7 @@ t2 1 t2_idx 1 b A 0 NULL NULL YES BTREE prepare stmt4 from ' show table status from test like ''t2%'' '; execute stmt4; Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment -t2 PBXT 10 Fixed 0 29 1 # 4096 0 NULL # # # latin1_swedish_ci NULL +t2 PBXT 10 Fixed 0 29 1024 # 4096 0 NULL # # # latin1_swedish_ci NULL prepare stmt4 from ' show table status from test like ''t9%'' '; execute stmt4; Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment === added file 'mysql-test/suite/pbxt/t/lowercase_table_grant-master.opt' --- a/mysql-test/suite/pbxt/t/lowercase_table_grant-master.opt 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/pbxt/t/lowercase_table_grant-master.opt 2009-08-31 11:07:44 +0000 @@ -0,0 +1 @@ +--lower_case_table_names === added file 'mysql-test/suite/pbxt/t/lowercase_table_qcache-master.opt' --- a/mysql-test/suite/pbxt/t/lowercase_table_qcache-master.opt 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/pbxt/t/lowercase_table_qcache-master.opt 2009-08-31 11:07:44 +0000 @@ -0,0 +1 @@ +--lower_case_table_names === added file 'mysql-test/suite/pbxt/t/lowercase_view-master.opt' --- a/mysql-test/suite/pbxt/t/lowercase_view-master.opt 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/pbxt/t/lowercase_view-master.opt 2009-08-31 11:07:44 +0000 @@ -0,0 +1 @@ +--lower_case_table_names=1 === added file 'mysql-test/suite/pbxt/t/udf-master.opt' --- a/mysql-test/suite/pbxt/t/udf-master.opt 1970-01-01 00:00:00 +0000 +++ b/mysql-test/suite/pbxt/t/udf-master.opt 2009-08-31 11:07:44 +0000 @@ -0,0 +1 @@ +$UDF_EXAMPLE_LIB_OPT === modified file 'storage/pbxt/src/Makefile.am' --- a/storage/pbxt/src/Makefile.am 2009-05-09 04:01:53 +0000 +++ b/storage/pbxt/src/Makefile.am 2009-08-31 11:07:44 +0000 @@ -19,7 +19,7 @@ noinst_HEADERS = bsearch_xt.h cache_xt. datadic_xt.h datalog_xt.h filesys_xt.h hashtab_xt.h \ ha_pbxt.h heap_xt.h index_xt.h linklist_xt.h \ memory_xt.h myxt_xt.h pthread_xt.h restart_xt.h \ - streaming_xt.h sortedlist_xt.h strutil_xt.h \ + pbms_enabled.h sortedlist_xt.h strutil_xt.h \ tabcache_xt.h table_xt.h trace_xt.h thread_xt.h \ util_xt.h xaction_xt.h xactlog_xt.h lock_xt.h \ systab_xt.h ha_xtsys.h discover_xt.h \ @@ -30,7 +30,7 @@ libpbxt_la_SOURCES = bsearch_xt.cc cache datadic_xt.cc datalog_xt.cc filesys_xt.cc hashtab_xt.cc \ ha_pbxt.cc heap_xt.cc index_xt.cc linklist_xt.cc \ memory_xt.cc myxt_xt.cc pthread_xt.cc restart_xt.cc \ - streaming_xt.cc sortedlist_xt.cc strutil_xt.cc \ + pbms_enabled.cc sortedlist_xt.cc strutil_xt.cc \ tabcache_xt.cc table_xt.cc trace_xt.cc thread_xt.cc \ systab_xt.cc ha_xtsys.cc discover_xt.cc \ util_xt.cc xaction_xt.cc xactlog_xt.cc lock_xt.cc locklist_xt.cc === modified file 'storage/pbxt/src/xt_config.h' --- a/storage/pbxt/src/xt_config.h 2009-08-18 07:46:53 +0000 +++ b/storage/pbxt/src/xt_config.h 2009-08-31 11:07:44 +0000 @@ -81,7 +81,8 @@ const int max_connections = 500; #define DEBUG #endif // _DEBUG #else -#define PBMS_ENABLED +// Paul suggested to disable PBMS in MariaDB for now. +// #define PBMS_ENABLED #endif #ifdef __FreeBSD__

1 0

[Maria-developers] [Branch ~maria-captains/maria/5.1] Rev 2721: Add missign header file to `make dist`.
by noreply＠launchpad.net 29 Aug '09

29 Aug '09

------------------------------------------------------------ revno: 2721 committer: knielsen(a)knielsen-hq.org branch nick: mariadb-5.1 timestamp: Sat 2009-08-29 21:48:04 +0200 message: Add missign header file to `make dist`. modified: storage/xtradb/Makefile.am -- lp:maria https://code.launchpad.net/~maria-captains/maria/5.1 Your team Maria developers is subscribed to branch lp:maria. To unsubscribe from this branch go to https://code.launchpad.net/~maria-captains/maria/5.1/+edit-subscription.

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2721)
by knielsen＠knielsen-hq.org 29 Aug '09

29 Aug '09

#At lp:maria 2721 knielsen(a)knielsen-hq.org 2009-08-29 Add missign header file to `make dist`. modified: storage/xtradb/Makefile.am === modified file 'storage/xtradb/Makefile.am' --- a/storage/xtradb/Makefile.am 2009-08-03 20:09:53 +0000 +++ b/storage/xtradb/Makefile.am 2009-08-29 19:48:04 +0000 @@ -131,7 +131,8 @@ noinst_HEADERS= include/btr0btr.h inclu include/ut0list.ic include/ut0wqueue.h \ include/ha_prototypes.h handler/ha_innodb.h \ include/handler0alter.h \ - handler/i_s.h handler/innodb_patch_info.h + handler/i_s.h handler/innodb_patch_info.h \ + handler/handler0vars.h EXTRA_LIBRARIES= libinnobase.a noinst_LIBRARIES= @plugin_innobase_static_target@

1 0

[Maria-developers] [Branch ~maria-captains/maria/5.1] Rev 2720: Fix `make dist` after solaris build file fixes.
by noreply＠launchpad.net 29 Aug '09

29 Aug '09

------------------------------------------------------------ revno: 2720 committer: knielsen(a)knielsen-hq.org branch nick: 5.1 timestamp: Sat 2009-08-29 21:04:46 +0200 message: Fix `make dist` after solaris build file fixes. Solaris build scripts under BUILD/ were changed, but Makefile.am was not updated to reflect the new set of file names. modified: BUILD/Makefile.am -- lp:maria https://code.launchpad.net/~maria-captains/maria/5.1 Your team Maria developers is subscribed to branch lp:maria. To unsubscribe from this branch go to https://code.launchpad.net/~maria-captains/maria/5.1/+edit-subscription.

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2720)
by knielsen＠knielsen-hq.org 29 Aug '09

29 Aug '09

#At lp:maria 2720 knielsen(a)knielsen-hq.org 2009-08-29 Fix `make dist` after solaris build file fixes. Solaris build scripts under BUILD/ were changed, but Makefile.am was not updated to reflect the new set of file names. modified: BUILD/Makefile.am === modified file 'BUILD/Makefile.am' --- a/BUILD/Makefile.am 2008-05-13 13:56:07 +0000 +++ b/BUILD/Makefile.am 2009-08-29 19:04:46 +0000 @@ -71,12 +71,16 @@ EXTRA_DIST = FINISH.sh \ compile-ppc-max \ compile-solaris-amd64 \ compile-solaris-amd64-debug \ + compile-solaris-amd64-debug-forte \ compile-solaris-amd64-forte \ - compile-solaris-amd64-forte-debug \ compile-solaris-sparc \ compile-solaris-sparc-debug \ compile-solaris-sparc-forte \ - compile-solaris-sparc-purify + compile-solaris-sparc-purify \ + compile-solaris-x86-32 \ + compile-solaris-x86-32-debug \ + compile-solaris-x86-32-debug-forte \ + compile-solaris-x86-forte-32 # Don't update the files from bitkeeper %::SCCS/s.%

1 0

[Maria-developers] Updated (by Guest): Backport key cache to 5.0 (52)
by worklog-noreply＠askmonty.org 28 Aug '09

28 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Backport key cache to 5.0 CREATION DATE..: Thu, 27 Aug 2009, 09:54 SUPERVISOR.....: Monty IMPLEMENTOR....: Monty COPIES TO......: Sanja CATEGORY.......: Server-Sprint TASK ID........: 52 (http://askmonty.org/worklog/?tid=52) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 6 ESTIMATE.......: 2 (hours remain) ORIG. ESTIMATE.: 8 PROGRESS NOTES: -=-=(Guest - Fri, 28 Aug 2009, 10:55)=-=- Low Level Design modified. --- /tmp/wklog.52.old.16945 2009-08-28 10:55:40.000000000 +0300 +++ /tmp/wklog.52.new.16945 2009-08-28 10:55:40.000000000 +0300 @@ -1 +1,22 @@ -Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. +Here is instructions of how to repeat this if needed: + +- Copy the following files from the latest MySQL 5.1 to MySQL 5.0: + +include/keycache.h +include/my_bit.h +mysys/mf_keycache.c +mysys/mf_keycaches.c +mysys/my_bit.c + +Apply the following change in mysys/mf_keycaches.c +81c81 +< static uchar *safe_hash_entry_get(SAFE_HASH_ENTRY *entry, size_t *length, +--- +> static uchar *safe_hash_entry_get(SAFE_HASH_ENTRY *entry, uint *length, + +recompile everything. + +Simple test to verify that this is working: +cd mysql-test +mysql-test-run --valgrind key_cache +(the --valgrind argument can be left out if you don't have valgrind installed) -=-=(Guest - Thu, 27 Aug 2009, 21:52)=-=- - Planning code (Email dicussions & discussions with Sanja): 2 hours. - Backport of code (2 hours). Notice that mysys/mf_keycaches.c::safe_hash_entry_get() is different from the code in MySQL 5.1 - Testing code with mysql-test-run (especially running the key_cache.test with valgrind) - Tested the server with sql-bench Worked 6 hours and estimate 2 hours remain (original estimate unchanged). -=-=(Sanja - Thu, 27 Aug 2009, 09:56)=-=- High-Level Specification modified. --- /tmp/wklog.52.old.20094 2009-08-27 09:56:05.000000000 +0300 +++ /tmp/wklog.52.new.20094 2009-08-27 09:56:05.000000000 +0300 @@ -1 +1 @@ - +Backport keycache to 5.0 -=-=(Sanja - Thu, 27 Aug 2009, 09:55)=-=- Low Level Design modified. --- /tmp/wklog.52.old.20035 2009-08-27 09:55:25.000000000 +0300 +++ /tmp/wklog.52.new.20035 2009-08-27 09:55:25.000000000 +0300 @@ -1 +1 @@ - +Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. DESCRIPTION: Backport keycache to bring bugfixes made in 5.1. HIGH-LEVEL SPECIFICATION: Backport keycache to 5.0 LOW-LEVEL DESIGN: Here is instructions of how to repeat this if needed: - Copy the following files from the latest MySQL 5.1 to MySQL 5.0: include/keycache.h include/my_bit.h mysys/mf_keycache.c mysys/mf_keycaches.c mysys/my_bit.c Apply the following change in mysys/mf_keycaches.c 81c81 < static uchar *safe_hash_entry_get(SAFE_HASH_ENTRY *entry, size_t *length, --- > static uchar *safe_hash_entry_get(SAFE_HASH_ENTRY *entry, uint *length, recompile everything. Simple test to verify that this is working: cd mysql-test mysql-test-run --valgrind key_cache (the --valgrind argument can be left out if you don't have valgrind installed) ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Backport key cache to 5.0 (52)
by worklog-noreply＠askmonty.org 28 Aug '09

28 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Backport key cache to 5.0 CREATION DATE..: Thu, 27 Aug 2009, 09:54 SUPERVISOR.....: Monty IMPLEMENTOR....: Monty COPIES TO......: Sanja CATEGORY.......: Server-Sprint TASK ID........: 52 (http://askmonty.org/worklog/?tid=52) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 6 ESTIMATE.......: 2 (hours remain) ORIG. ESTIMATE.: 8 PROGRESS NOTES: -=-=(Guest - Fri, 28 Aug 2009, 10:55)=-=- Low Level Design modified. --- /tmp/wklog.52.old.16945 2009-08-28 10:55:40.000000000 +0300 +++ /tmp/wklog.52.new.16945 2009-08-28 10:55:40.000000000 +0300 @@ -1 +1,22 @@ -Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. +Here is instructions of how to repeat this if needed: + +- Copy the following files from the latest MySQL 5.1 to MySQL 5.0: + +include/keycache.h +include/my_bit.h +mysys/mf_keycache.c +mysys/mf_keycaches.c +mysys/my_bit.c + +Apply the following change in mysys/mf_keycaches.c +81c81 +< static uchar *safe_hash_entry_get(SAFE_HASH_ENTRY *entry, size_t *length, +--- +> static uchar *safe_hash_entry_get(SAFE_HASH_ENTRY *entry, uint *length, + +recompile everything. + +Simple test to verify that this is working: +cd mysql-test +mysql-test-run --valgrind key_cache +(the --valgrind argument can be left out if you don't have valgrind installed) -=-=(Guest - Thu, 27 Aug 2009, 21:52)=-=- - Planning code (Email dicussions & discussions with Sanja): 2 hours. - Backport of code (2 hours). Notice that mysys/mf_keycaches.c::safe_hash_entry_get() is different from the code in MySQL 5.1 - Testing code with mysql-test-run (especially running the key_cache.test with valgrind) - Tested the server with sql-bench Worked 6 hours and estimate 2 hours remain (original estimate unchanged). -=-=(Sanja - Thu, 27 Aug 2009, 09:56)=-=- High-Level Specification modified. --- /tmp/wklog.52.old.20094 2009-08-27 09:56:05.000000000 +0300 +++ /tmp/wklog.52.new.20094 2009-08-27 09:56:05.000000000 +0300 @@ -1 +1 @@ - +Backport keycache to 5.0 -=-=(Sanja - Thu, 27 Aug 2009, 09:55)=-=- Low Level Design modified. --- /tmp/wklog.52.old.20035 2009-08-27 09:55:25.000000000 +0300 +++ /tmp/wklog.52.new.20035 2009-08-27 09:55:25.000000000 +0300 @@ -1 +1 @@ - +Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. DESCRIPTION: Backport keycache to bring bugfixes made in 5.1. HIGH-LEVEL SPECIFICATION: Backport keycache to 5.0 LOW-LEVEL DESIGN: Here is instructions of how to repeat this if needed: - Copy the following files from the latest MySQL 5.1 to MySQL 5.0: include/keycache.h include/my_bit.h mysys/mf_keycache.c mysys/mf_keycaches.c mysys/my_bit.c Apply the following change in mysys/mf_keycaches.c 81c81 < static uchar *safe_hash_entry_get(SAFE_HASH_ENTRY *entry, size_t *length, --- > static uchar *safe_hash_entry_get(SAFE_HASH_ENTRY *entry, uint *length, recompile everything. Simple test to verify that this is working: cd mysql-test mysql-test-run --valgrind key_cache (the --valgrind argument can be left out if you don't have valgrind installed) ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Simple compiler warning fixes for yassl-1.9.6
by Kristian Nielsen 28 Aug '09

28 Aug '09

The attached patch contains a few simple fixes for compiler warnings that I applied to our copy of YaSSL in the MariaDB source tree (http://askmonty.org/wiki/index.php/MariaDB) I am sending you these in case you want to include them in upstream YaSSL. I did not see instructions for submitting patches at http://www.yassl.com/, so please let me know if there is some other email address or procedure I should follow. Signed-off-by: Kristian Nielsen <knielsen(a)knielsen-hq.org>

2 1

Re: [Maria-developers] QC patch for review
by Michael Widenius 27 Aug '09

27 Aug '09

Hi! This is regarding a bug fix for query cache: - Bug#39249 Maria: query cache returns out-of-date results - Bug #41098 Query Cache returns wrong result with concurrent insert >>>>> "s" == Oleksandr Byelkin <Oleksandr> writes: s> Hi! s> Here it is QC patch for inserts (as we discussed) and also there is DBUG s> problem workaround in the test. s> +++ sql/handler.h 2009-06-11 11:25:58 +0000 s> @@ -247,6 +247,11 @@ s> #define HA_CACHE_TBL_NOCACHE 1 s> #define HA_CACHE_TBL_ASKTRANSACT 2 s> #define HA_CACHE_TBL_TRANSACT 4 s> +/** s> + Non transactional table but insert results visible for other threads s> + only on unlock s> +*/ s> +#define HA_CACHE_TBL_NTRNS_INS2LOCK 8 The above is true for all non transactional engines. To clarify, there are two types of non transactional engines when it comes to insert: 1) The table is locked at start of insert statement and can't be accessed by other threads until unlock. 2) The table is locked at start of insert statement, but other threads can read only the original rows until unlock, after which all rows are visible (myisam/maria with concurrent insert on) In both cases, the new rows are visible after unlock, in which case we shouldn't need the above flag. As we discussed in Majorca, I think the logic for this patch is wrong. The current patch works (as I can see) on the following idea: - mark non-transacitonal tables that are part of an insert - later in sql_base.cc::close_thread_table() call query_cache_invalidate4() to invalidate the table if there was a possible 'concurrent insert'. The reason the above doesn't work is that another thread may have done a lock on the table before the unlock happens in the concurrent-inserted thread and then call 'query_cache_store_query()' after the invalidate is done. In this case the other thread has and old view of the table and anything it stores in the cache will be 'old'. The new code only makes wrong things 'less unlikely' to happen; It doesn't remove the original bug. The reason for the problem with concurrent insert problem in the query cache are two: - The query cache should get invalidated at once when table changes state (ie, when the new table content is visible for another thread). This should be done under a mutex to ensure that there is no race condition for this table. - When validating of select with a table can be put into the query cache we should ensure that it's content is of the latest version. With the two above axioms in place, the query cache will be safe for concurrent inserts. There are some other benefits of changing to use this model: - We can safely use locked tables with the query cache. - We can put the query into the table cache even under lock tables, as long as all the tables are marked as 'cacheable'. - We can use the data from the query cache, as long as all locked tables are marked as 'cacheable'. - We could (later) merge the code of transactional and not transactional tables as they act the same. - We can provide caching for versioned tables (like Maria) more easily. Here is a last comment about the problem with this patch: s> === modified file 'sql/sql_base.cc' s> --- sql/sql_base.cc 2009-05-19 09:28:05 +0000 s> +++ sql/sql_base.cc 2009-06-11 11:25:58 +0000 s> @@ -1373,6 +1373,15 @@ bool close_thread_table(THD *thd, TABLE s> table->s->table_name.str, (long) table)); s> *table_ptr=table->next; s> + s> + /* Invalidate if it has mark about changing in insert s> + (not all tables has such marks */ Fix comment style and add missing brace + if (table->changed_in_insert) + { + table->changed_in_insert= FALSE; + query_cache_invalidate4(thd, table, FALSE, FALSE); + } I have some problems with the above code: - The real problem is that if someone accesses the table between the invalidate() and unlock, they see the old version of the table and not the real rows. The above change adds an extra invalidate in case of concurrent inserts. However, another thread may already have done the following before this call: - Do a query just after invalidate, in which case the table is seen as 'old' - Do another query after unlock, in which case table is seen as 'new'. - Another problem is that another thread that did a lock table before the invalidate, it will still see the old data from the table. If this thread does a query of the table and it goes into the query cache, then the query cache will be filled with wrong data. (Note that even if we don't put tables in the cache under lock tables, the problem is still the same for tables without lock tables. It's just easier to simulate the problem if we test things with lock tables) - A future problem is that for nontransactional tables that could do concurrent deletes/updates the above wouldn't work. So this code only makes the problem less likely, it doesn't fix it. ---------------------- Add to MyISAM a function: virtual my_bool register_query_cache_table(...) { /* check if this thread is seeing all rows in the file */ if (file->s->state.data_file_length != file->state->data_file_length) { /* Some thread (or even this thread) has inserted more rows into the table since it was locked; Don't use the query cache. */ return 0; } return 1; } We also need to invalidate tables when they change. This should be done by adding a callback in mi_locking.c:mi_update_status() to invalidate the query cache for any entries for this table. (This only needs to be done inside 'if (info->state == &info->save_state)' In Maira we need to to the same changes in _ma_update_status(). In Maria, we can also decide to handle transactional/versioned tables with the query cache: virtual my_bool register_query_cache_table(....) { .... if (share->now_transactional && share->have_versioning) return (file->trn != file->s->state.last_change_trn); ... } For the above to work, we need to set last_change_trn in _ma_trnman_end_trans_hook() when we store the state for a changed table (at same place where we update 'records' and 'checksum') Regards, Monty PS: Don't forget to also fix the MERGE tables and check that the Maria bug is truly fixed.

1 0

[Maria-developers] Progress (by Guest): Backport key cache to 5.0 (52)
by worklog-noreply＠askmonty.org 27 Aug '09

27 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Backport key cache to 5.0 CREATION DATE..: Thu, 27 Aug 2009, 09:54 SUPERVISOR.....: Monty IMPLEMENTOR....: Monty COPIES TO......: Sanja CATEGORY.......: Server-Sprint TASK ID........: 52 (http://askmonty.org/worklog/?tid=52) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 6 ESTIMATE.......: 2 (hours remain) ORIG. ESTIMATE.: 8 PROGRESS NOTES: -=-=(Guest - Thu, 27 Aug 2009, 21:52)=-=- - Planning code (Email dicussions & discussions with Sanja): 2 hours. - Backport of code (2 hours). Notice that mysys/mf_keycaches.c::safe_hash_entry_get() is different from the code in MySQL 5.1 - Testing code with mysql-test-run (especially running the key_cache.test with valgrind) - Tested the server with sql-bench Worked 6 hours and estimate 2 hours remain (original estimate unchanged). -=-=(Sanja - Thu, 27 Aug 2009, 09:56)=-=- High-Level Specification modified. --- /tmp/wklog.52.old.20094 2009-08-27 09:56:05.000000000 +0300 +++ /tmp/wklog.52.new.20094 2009-08-27 09:56:05.000000000 +0300 @@ -1 +1 @@ - +Backport keycache to 5.0 -=-=(Sanja - Thu, 27 Aug 2009, 09:55)=-=- Low Level Design modified. --- /tmp/wklog.52.old.20035 2009-08-27 09:55:25.000000000 +0300 +++ /tmp/wklog.52.new.20035 2009-08-27 09:55:25.000000000 +0300 @@ -1 +1 @@ - +Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. DESCRIPTION: Backport keycache to bring bugfixes made in 5.1. HIGH-LEVEL SPECIFICATION: Backport keycache to 5.0 LOW-LEVEL DESIGN: Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Progress (by Guest): Backport key cache to 5.0 (52)
by worklog-noreply＠askmonty.org 27 Aug '09

27 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Backport key cache to 5.0 CREATION DATE..: Thu, 27 Aug 2009, 09:54 SUPERVISOR.....: Monty IMPLEMENTOR....: Monty COPIES TO......: Sanja CATEGORY.......: Server-Sprint TASK ID........: 52 (http://askmonty.org/worklog/?tid=52) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 6 ESTIMATE.......: 2 (hours remain) ORIG. ESTIMATE.: 8 PROGRESS NOTES: -=-=(Guest - Thu, 27 Aug 2009, 21:52)=-=- - Planning code (Email dicussions & discussions with Sanja): 2 hours. - Backport of code (2 hours). Notice that mysys/mf_keycaches.c::safe_hash_entry_get() is different from the code in MySQL 5.1 - Testing code with mysql-test-run (especially running the key_cache.test with valgrind) - Tested the server with sql-bench Worked 6 hours and estimate 2 hours remain (original estimate unchanged). -=-=(Sanja - Thu, 27 Aug 2009, 09:56)=-=- High-Level Specification modified. --- /tmp/wklog.52.old.20094 2009-08-27 09:56:05.000000000 +0300 +++ /tmp/wklog.52.new.20094 2009-08-27 09:56:05.000000000 +0300 @@ -1 +1 @@ - +Backport keycache to 5.0 -=-=(Sanja - Thu, 27 Aug 2009, 09:55)=-=- Low Level Design modified. --- /tmp/wklog.52.old.20035 2009-08-27 09:55:25.000000000 +0300 +++ /tmp/wklog.52.new.20035 2009-08-27 09:55:25.000000000 +0300 @@ -1 +1 @@ - +Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. DESCRIPTION: Backport keycache to bring bugfixes made in 5.1. HIGH-LEVEL SPECIFICATION: Backport keycache to 5.0 LOW-LEVEL DESIGN: Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] MariaDB 5.1 release coordinator
by Henrik Ingo 27 Aug '09

27 Aug '09

Hi all Just finished a writeup of our meeting last week, on http://openlife.cc/blogs/2009/august/mariadb-release-plan-and-other-news-mp… (Btw, I've also uploaded slides to the MP internal wiki and updated the ProjectHandbook based on our discussion.) Then comes the final thing on my todo from discussions in Mallorca: Sergey P suggested we should appoint a release coordinator to chase all the tasks needed before releasing beta/RC/GA of MariaDB: http://askmonty.org/wiki/index.php/MariaDB_versus_MySQL#Things_to_add_to_Ma… http://askmonty.org/wiki/index.php/MariaDB51_release_plan he also volunteered, and doesn't have customer projects coming up, and Monty approved, so Sergey: you're it! This means - You should keep especially Sergey updated to whatever you contribute to release progress. - You should be able to ask Sergey any questions on status of tasks, should we possibly drop a feature etc... - Public channels (this mailing list, IRC) still preferred - Continue as usual, this doesn't mean that Sergey will go around asking people to do work. Sergey, please continue the list if you had something in mind that's not there. henrik -- email: henrik.ingo(a)avoinelama.fi tel: +358-40-5697354 www: www.avoinelama.fi/~hingo book: www.openlife.cc

1 0

Re: [Maria-developers] Fixing compiler warnings
by Kristian Nielsen 27 Aug '09

27 Aug '09

Paul McCullagh <paul.mccullagh(a)primebase.org> writes: > Hi Kristian, > > Thanks for the patch! > > I will definitely include these changes in the PBXT trunk. > > I am happy to accept any changes that allow the code to compile > without warnings (this is also a requirement for compiling with > Drizzle). The other warning I saw in PBXT was this: cache_xt.h:48: warning: ‘typedef’ was ignored in this declaration cache_xt.h:49: warning: ‘typedef’ was ignored in this declaration The code in question is this: typedef enum XTPageLockType { XT_LOCK_READ, XT_LOCK_WRITE, XT_XLOCK_LEAF }; typedef enum XTPageUnlockType { XT_UNLOCK_NONE, XT_UNLOCK_READ, XT_UNLOCK_WRITE, XT_UNLOCK_R_UPDATE, XT_UNLOCK_W_UPDATE }; When I first looked I thought this was due to mixing C and C++, and thought that the compiler was being stupid. But now I see that it is actually the code that is a bit silly, as it says typedef without giving a name of the type being defined! (So now I'm surprised that we don't get a syntax error, but I guess this is just one of those dark corners of C++ that are best left alone ...) Anyway, mentioning it in case you want to fix. - Kristian.

2 1

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2720)
by knielsen＠knielsen-hq.org 27 Aug '09

27 Aug '09

#At lp:maria 2720 knielsen(a)knielsen-hq.org 2009-08-27 Fix parsing of invalid plugin enum option value. Previous patch to fix plugin enum option parsing on big-endian introduced another bug due to incorrect comparison of unsigned value. This would cause an incorrect value to be parsed as value 0. See also MySQL BUG#41010 and BUG#32034. added: mysql-test/r/plugin_load2.result mysql-test/t/plugin_load2-master.opt mysql-test/t/plugin_load2.test modified: mysql-test/valgrind.supp mysys/my_getopt.c per-file messages: mysql-test/r/plugin_load2.result Add test case. mysql-test/t/plugin_load2-master.opt Add test case. mysql-test/t/plugin_load2.test Add test case. mysql-test/valgrind.supp Make dlclose() suppression catch also another possible call path, seen in new test case. mysys/my_getopt.c Fix incorrect comparison of unsigned value. === added file 'mysql-test/r/plugin_load2.result' --- a/mysql-test/r/plugin_load2.result 1970-01-01 00:00:00 +0000 +++ b/mysql-test/r/plugin_load2.result 2009-08-27 09:42:25 +0000 @@ -0,0 +1,2 @@ +SELECT @@global.example_enum_var = 'e2'; +ERROR HY000: Unknown system variable 'example_enum_var' === added file 'mysql-test/t/plugin_load2-master.opt' --- a/mysql-test/t/plugin_load2-master.opt 1970-01-01 00:00:00 +0000 +++ b/mysql-test/t/plugin_load2-master.opt 2009-08-27 09:42:25 +0000 @@ -0,0 +1,3 @@ +$EXAMPLE_PLUGIN_OPT +"--plugin-load=;EXAMPLE=ha_example.so;" +--loose-plugin-example-enum-var=nonexistientvalue === added file 'mysql-test/t/plugin_load2.test' --- a/mysql-test/t/plugin_load2.test 1970-01-01 00:00:00 +0000 +++ b/mysql-test/t/plugin_load2.test 2009-08-27 09:42:25 +0000 @@ -0,0 +1,12 @@ +--source include/have_example_plugin.inc + +# Test for bug in parsing plugin enum option. +# The bug was that the error from parsing a non-existent value was not properly +# handled, so the variable was assigned some arbitrary wrong value. +# +# We test this by passing --loose-plugin-example-enum-var=nonexistientvalue in +# the .opt file of the test case, and check that the variable is not wrongly +# set to a value in this case. + +--error 1193 +SELECT @@global.example_enum_var = 'e2'; === modified file 'mysql-test/valgrind.supp' --- a/mysql-test/valgrind.supp 2009-08-05 07:21:37 +0000 +++ b/mysql-test/valgrind.supp 2009-08-27 09:42:25 +0000 @@ -387,6 +387,12 @@ fun:plugin_dl_del(st_mysql_lex_string const*) } +# +# Glibc _dl_close_worker() re-allocates a scope data structure, and frees the +# old one. This isn't a leak, but generates "still reachable" warnings, as +# there is no global destructor code to do a final free() at exit(). +# + { dlclose memory loss from plugin variant 2 Memcheck:Leak @@ -397,7 +403,6 @@ fun:_dlerror_run fun:dlclose fun:_Z15free_plugin_memP12st_plugin_dl - fun:_Z13plugin_dl_delPK19st_mysql_lex_string } { @@ -411,7 +416,6 @@ fun:_dlerror_run fun:dlclose fun:_Z15free_plugin_memP12st_plugin_dl - fun:_Z13plugin_dl_delPK19st_mysql_lex_string } { @@ -424,7 +428,6 @@ obj:/lib*/libdl-*.so fun:dlclose fun:_ZL15free_plugin_memP12st_plugin_dl - fun:_ZL13plugin_dl_delPK19st_mysql_lex_string } { @@ -452,7 +455,6 @@ fun:_dlerror_run fun:dlclose fun:_ZL15free_plugin_memP12st_plugin_dl - fun:_ZL13plugin_dl_delPK19st_mysql_lex_string } { @@ -466,7 +468,6 @@ fun:_dlerror_run fun:dlclose fun:_ZL15free_plugin_memP12st_plugin_dl - fun:_ZL13plugin_dl_delPK19st_mysql_lex_string } { === modified file 'mysys/my_getopt.c' --- a/mysys/my_getopt.c 2009-05-20 15:34:34 +0000 +++ b/mysys/my_getopt.c 2009-08-27 09:42:25 +0000 @@ -603,6 +603,7 @@ static int setval(const struct my_option my_bool set_maximum_value) { int err= 0; + int pos; if (value && argument) { @@ -647,7 +648,9 @@ static int setval(const struct my_option return EXIT_OUT_OF_MEMORY; break; case GET_ENUM: - if (((*(ulong *)result_pos)= find_type(argument, opts->typelib, 2) - 1) < 0) + pos = find_type(argument, opts->typelib, 2) - 1; + (*(ulong *)result_pos)= pos; + if (pos < 0) return EXIT_ARGUMENT_INVALID; break; case GET_SET:

2 1

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2721)
by knielsen＠knielsen-hq.org 27 Aug '09

27 Aug '09

#At lp:maria 2721 knielsen(a)knielsen-hq.org 2009-08-27 Fix most Compiler warnings seen in buildbot. Also add suppressions for a few warnings that cannot be meaningfully fixed by MariaDB developers. Changes for XtraDB, PBXT, and YaSSL also submitted upstream. added: support-files/ccfilter modified: client/mysqlbinlog.cc config/ac-macros/misc.m4 extra/yassl/include/yassl_int.hpp extra/yassl/src/handshake.cpp extra/yassl/src/yassl_imp.cpp extra/yassl/src/yassl_int.cpp extra/yassl/taocrypt/include/modes.hpp extra/yassl/taocrypt/src/asn.cpp mysys/my_compress.c sql/mysqld.cc sql/strfunc.cc storage/pbxt/src/datadic_xt.cc storage/pbxt/src/discover_xt.cc storage/xtradb/fil/fil0fil.c storage/xtradb/mtr/mtr0mtr.c storage/xtradb/srv/srv0srv.c storage/xtradb/srv/srv0start.c strings/decimal.c support-files/compiler_warnings.supp per-file messages: client/mysqlbinlog.cc Fix compiler warnings. config/ac-macros/misc.m4 Fix wrong naming, autoconfig requires _cv_ in cached names. extra/yassl/include/yassl_int.hpp Fix compiler warnings. extra/yassl/src/handshake.cpp Fix compiler warnings. extra/yassl/src/yassl_imp.cpp Fix compiler warnings. extra/yassl/src/yassl_int.cpp Fix compiler warnings. extra/yassl/taocrypt/include/modes.hpp Fix compiler warnings. extra/yassl/taocrypt/src/asn.cpp Fix compiler warnings. mysys/my_compress.c Fix compiler warnings. sql/mysqld.cc Fix compiler warnings. sql/strfunc.cc Fix compiler warnings. storage/pbxt/src/datadic_xt.cc Fix compiler warnings. storage/pbxt/src/discover_xt.cc Fix compiler warnings. storage/xtradb/fil/fil0fil.c Fix compiler warnings. storage/xtradb/mtr/mtr0mtr.c Fix compiler warnings. storage/xtradb/srv/srv0srv.c Fix compiler warnings. storage/xtradb/srv/srv0start.c Fix compiler warnings. strings/decimal.c Fix compiler warnings. support-files/ccfilter Add helper for suppressing compiler warnings in local developer source tree. Allows to check for not introducing new warnings into Buildbot without having to actually run the build through Buildbot. support-files/compiler_warnings.supp Suppress a few warnings that cannot be meaningfully fixed in source code. === modified file 'client/mysqlbinlog.cc' --- a/client/mysqlbinlog.cc 2009-02-13 16:41:47 +0000 +++ b/client/mysqlbinlog.cc 2009-08-27 09:48:29 +0000 @@ -689,8 +689,8 @@ Exit_status process_event(PRINT_EVENT_IN Format events are not concerned by --offset and such, we always need to read them to be able to process the wanted events. */ - if ((rec_count >= offset) && - ((my_time_t)(ev->when) >= start_datetime) || + if (((rec_count >= offset) && + ((my_time_t)(ev->when) >= start_datetime)) || (ev_type == FORMAT_DESCRIPTION_EVENT)) { if (ev_type != FORMAT_DESCRIPTION_EVENT) === modified file 'config/ac-macros/misc.m4' --- a/config/ac-macros/misc.m4 2008-02-26 17:38:43 +0000 +++ b/config/ac-macros/misc.m4 2009-08-27 09:48:29 +0000 @@ -594,15 +594,15 @@ dnl ------------------------------------ dnl MYSQL_NEEDS_MYSYS_NEW AC_DEFUN([MYSQL_NEEDS_MYSYS_NEW], -[AC_CACHE_CHECK([needs mysys_new helpers], mysql_use_mysys_new, +[AC_CACHE_CHECK([needs mysys_new helpers], mysql_cv_use_mysys_new, [ AC_LANG_PUSH(C++) AC_TRY_LINK([], [ class A { public: int b; }; A *a=new A; a->b=10; delete a; -], mysql_use_mysys_new=no, mysql_use_mysys_new=yes) +], mysql_cv_use_mysys_new=no, mysql_cv_use_mysys_new=yes) AC_LANG_POP(C++) ]) -if test "$mysql_use_mysys_new" = "yes" +if test "$mysql_cv_use_mysys_new" = "yes" then AC_DEFINE([USE_MYSYS_NEW], [1], [Needs to use mysys_new helpers]) fi === modified file 'extra/yassl/include/yassl_int.hpp' --- a/extra/yassl/include/yassl_int.hpp 2008-11-18 16:45:44 +0000 +++ b/extra/yassl/include/yassl_int.hpp 2009-08-27 09:48:29 +0000 @@ -441,7 +441,7 @@ public: const Ciphers& GetCiphers() const; const DH_Parms& GetDH_Parms() const; const Stats& GetStats() const; - const VerifyCallback getVerifyCallback() const; + VerifyCallback getVerifyCallback() const; pem_password_cb GetPasswordCb() const; void* GetUserData() const; bool GetSessionCacheOff() const; === modified file 'extra/yassl/src/handshake.cpp' --- a/extra/yassl/src/handshake.cpp 2008-04-28 16:24:05 +0000 +++ b/extra/yassl/src/handshake.cpp 2009-08-27 09:48:29 +0000 @@ -789,7 +789,7 @@ void processReply(SSL& ssl) { if (ssl.GetError()) return; - if (DoProcessReply(ssl)) + if (DoProcessReply(ssl)) { // didn't complete process if (!ssl.getSocket().IsNonBlocking()) { // keep trying now, blocking ok @@ -799,6 +799,7 @@ void processReply(SSL& ssl) else // user will have try again later, non blocking ssl.SetError(YasslError(SSL_ERROR_WANT_READ)); + } } @@ -872,11 +873,12 @@ void sendServerKeyExchange(SSL& ssl, Buf // send change cipher void sendChangeCipher(SSL& ssl, BufferOutput buffer) { - if (ssl.getSecurity().get_parms().entity_ == server_end) + if (ssl.getSecurity().get_parms().entity_ == server_end) { if (ssl.getSecurity().get_resuming()) ssl.verifyState(clientKeyExchangeComplete); else ssl.verifyState(clientFinishedComplete); + } if (ssl.GetError()) return; ChangeCipherSpec ccs; === modified file 'extra/yassl/src/yassl_imp.cpp' --- a/extra/yassl/src/yassl_imp.cpp 2009-02-13 16:41:47 +0000 +++ b/extra/yassl/src/yassl_imp.cpp 2009-08-27 09:48:29 +0000 @@ -1304,7 +1304,7 @@ void ServerHello::Process(input_buffer&, else ssl.useSecurity().use_connection().sessionID_Set_ = false; - if (ssl.getSecurity().get_resuming()) + if (ssl.getSecurity().get_resuming()) { if (memcmp(session_id_, ssl.getSecurity().get_resume().GetID(), ID_LEN) == 0) { ssl.set_masterSecret(ssl.getSecurity().get_resume().GetSecret()); @@ -1319,6 +1319,7 @@ void ServerHello::Process(input_buffer&, ssl.useSecurity().set_resuming(false); ssl.useLog().Trace("server denied resumption"); } + } if (ssl.CompressionOn() && !compression_method_) ssl.UnSetCompression(); // server isn't supporting yaSSL zlib request === modified file 'extra/yassl/src/yassl_int.cpp' --- a/extra/yassl/src/yassl_int.cpp 2009-02-13 16:41:47 +0000 +++ b/extra/yassl/src/yassl_int.cpp 2009-08-27 09:48:29 +0000 @@ -1833,7 +1833,7 @@ SSL_CTX::GetCA_List() const } -const VerifyCallback SSL_CTX::getVerifyCallback() const +VerifyCallback SSL_CTX::getVerifyCallback() const { return verifyCallback_; } === modified file 'extra/yassl/taocrypt/include/modes.hpp' --- a/extra/yassl/taocrypt/include/modes.hpp 2007-03-23 12:43:09 +0000 +++ b/extra/yassl/taocrypt/include/modes.hpp 2009-08-27 09:48:29 +0000 @@ -95,11 +95,12 @@ inline void Mode_BASE::Process(byte* out { if (mode_ == ECB) ECB_Process(out, in, sz); - else if (mode_ == CBC) + else if (mode_ == CBC) { if (dir_ == ENCRYPTION) CBC_Encrypt(out, in, sz); else CBC_Decrypt(out, in, sz); + } } === modified file 'extra/yassl/taocrypt/src/asn.cpp' --- a/extra/yassl/taocrypt/src/asn.cpp 2009-02-13 16:41:47 +0000 +++ b/extra/yassl/taocrypt/src/asn.cpp 2009-08-27 09:48:29 +0000 @@ -780,11 +780,12 @@ void CertDecoder::GetDate(DateType dt) memcpy(date, source_.get_current(), length); source_.advance(length); - if (!ValidateDate(date, b, dt) && verify_) + if (!ValidateDate(date, b, dt) && verify_) { if (dt == BEFORE) source_.SetError(BEFORE_DATE_E); else source_.SetError(AFTER_DATE_E); + } // save for later use if (dt == BEFORE) { @@ -1061,7 +1062,7 @@ word32 DecodeDSA_Signature(byte* decoded return 0; } word32 rLen = GetLength(source); - if (rLen != 20) + if (rLen != 20) { if (rLen == 21) { // zero at front, eat source.next(); --rLen; @@ -1074,6 +1075,7 @@ word32 DecodeDSA_Signature(byte* decoded source.SetError(DSA_SZ_E); return 0; } + } memcpy(decoded, source.get_buffer() + source.get_index(), rLen); source.advance(rLen); @@ -1083,7 +1085,7 @@ word32 DecodeDSA_Signature(byte* decoded return 0; } word32 sLen = GetLength(source); - if (sLen != 20) + if (sLen != 20) { if (sLen == 21) { source.next(); // zero at front, eat --sLen; @@ -1096,6 +1098,7 @@ word32 DecodeDSA_Signature(byte* decoded source.SetError(DSA_SZ_E); return 0; } + } memcpy(decoded + rLen, source.get_buffer() + source.get_index(), sLen); source.advance(sLen); === modified file 'mysys/my_compress.c' --- a/mysys/my_compress.c 2009-05-22 12:38:50 +0000 +++ b/mysys/my_compress.c 2009-08-27 09:48:29 +0000 @@ -81,12 +81,13 @@ my_bool my_compress(uchar *packet, size_ This fix is safe, since such memory is only used internally by zlib, so we will not hide any bugs in mysql this way. */ -void *my_az_allocator(void *dummy, unsigned int items, unsigned int size) +void *my_az_allocator(void *dummy __attribute__((unused)), unsigned int items, + unsigned int size) { return my_malloc((size_t)items*(size_t)size, IF_VALGRIND(MY_ZEROFILL, MYF(0))); } -void my_az_free(void *dummy, void *address) +void my_az_free(void *dummy __attribute__((unused)), void *address) { my_free(address, MYF(MY_ALLOW_ZERO_PTR)); } === modified file 'sql/mysqld.cc' --- a/sql/mysqld.cc 2009-05-19 09:28:05 +0000 +++ b/sql/mysqld.cc 2009-08-27 09:48:29 +0000 @@ -1002,6 +1002,7 @@ static void close_connections(void) } +#ifdef HAVE_CLOSE_SERVER_SOCK static void close_socket(my_socket sock, const char *info) { DBUG_ENTER("close_socket"); @@ -1021,6 +1022,7 @@ static void close_socket(my_socket sock, } DBUG_VOID_RETURN; } +#endif static void close_server_sock() === modified file 'sql/strfunc.cc' --- a/sql/strfunc.cc 2009-04-25 10:05:32 +0000 +++ b/sql/strfunc.cc 2009-08-27 09:48:29 +0000 @@ -148,7 +148,7 @@ static uint parse_name(TYPELIB *lib, con } } else - for (; pos != end && *pos != '=' && *pos !=',' ; pos++); + for (; pos != end && *pos != '=' && *pos !=',' ; pos++) {} uint var_len= (uint) (pos - start); /* Determine which flag it is */ === modified file 'storage/pbxt/src/datadic_xt.cc' --- a/storage/pbxt/src/datadic_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/datadic_xt.cc 2009-08-27 09:48:29 +0000 @@ -2871,5 +2871,5 @@ xtBool XTDDTable::checkCanDrop() { /* no refs or references only itself */ return (dt_trefs == NULL) || - (dt_trefs->tr_next == NULL) && (dt_trefs->tr_fkey->co_table == this); + ((dt_trefs->tr_next == NULL) && (dt_trefs->tr_fkey->co_table == this)); } === modified file 'storage/pbxt/src/discover_xt.cc' --- a/storage/pbxt/src/discover_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/discover_xt.cc 2009-08-27 09:48:29 +0000 @@ -493,8 +493,8 @@ mysql_prepare_create_table(THD *thd, HA_ } /* Don't pack rows in old tables if the user has requested this */ if ((sql_field->flags & BLOB_FLAG) || - sql_field->sql_type == MYSQL_TYPE_VARCHAR && - create_info->row_type != ROW_TYPE_FIXED) + (sql_field->sql_type == MYSQL_TYPE_VARCHAR && + create_info->row_type != ROW_TYPE_FIXED)) (*db_options)|= HA_OPTION_PACK_RECORD; it2.rewind(); } @@ -963,7 +963,7 @@ mysql_prepare_create_table(THD *thd, HA_ sql_field->sql_type == MYSQL_TYPE_VARCHAR || sql_field->pack_flag & FIELDFLAG_BLOB))) { - if (column_nr == 0 && (sql_field->pack_flag & FIELDFLAG_BLOB) || + if ((column_nr == 0 && (sql_field->pack_flag & FIELDFLAG_BLOB)) || sql_field->sql_type == MYSQL_TYPE_VARCHAR) key_info->flags|= HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY; else === modified file 'storage/xtradb/fil/fil0fil.c' --- a/storage/xtradb/fil/fil0fil.c 2009-06-25 01:43:25 +0000 +++ b/storage/xtradb/fil/fil0fil.c 2009-08-27 09:48:29 +0000 @@ -45,7 +45,9 @@ Created 10/25/1995 Heikki Tuuri #include "trx0trx.h" #include "trx0sys.h" #include "pars0pars.h" +#include "row0row.h" #include "row0mysql.h" +#include "que0que.h" /* @@ -3134,7 +3136,7 @@ skip_info: rec_offs_init(offsets_); - fprintf(stderr, "InnoDB: Progress in %:"); + fprintf(stderr, "%s", "InnoDB: Progress in %:"); for (offset = 0; offset < size_bytes; offset += UNIV_PAGE_SIZE) { success = os_file_read(file, page, === modified file 'storage/xtradb/mtr/mtr0mtr.c' --- a/storage/xtradb/mtr/mtr0mtr.c 2009-06-25 01:43:25 +0000 +++ b/storage/xtradb/mtr/mtr0mtr.c 2009-08-27 09:48:29 +0000 @@ -32,6 +32,7 @@ Created 11/26/1995 Heikki Tuuri #include "page0types.h" #include "mtr0log.h" #include "log0log.h" +#include "buf0flu.h" /********************************************************************* Releases the item in the slot given. */ === modified file 'storage/xtradb/srv/srv0srv.c' --- a/storage/xtradb/srv/srv0srv.c 2009-07-06 05:47:15 +0000 +++ b/storage/xtradb/srv/srv0srv.c 2009-08-27 09:48:29 +0000 @@ -1815,7 +1815,6 @@ srv_printf_innodb_monitor( ulint btr_search_sys_subtotal; ulint lock_sys_subtotal; ulint recv_sys_subtotal; - ulint io_counter_subtotal; ulint i; trx_t* trx; === modified file 'storage/xtradb/srv/srv0start.c' --- a/storage/xtradb/srv/srv0start.c 2009-08-03 20:09:53 +0000 +++ b/storage/xtradb/srv/srv0start.c 2009-08-27 09:48:29 +0000 @@ -122,20 +122,6 @@ static char* srv_monitor_file_name; #define SRV_MAX_N_PENDING_SYNC_IOS 100 -/* Avoid warnings when using purify */ - -#ifdef HAVE_valgrind -static int inno_bcmp(register const char *s1, register const char *s2, - register uint len) -{ - while ((len-- != 0) && (*s1++ == *s2++)) - ; - - return(len + 1); -} -#define memcmp(A,B,C) inno_bcmp((A),(B),(C)) -#endif - static char* srv_parse_megabytes( === modified file 'strings/decimal.c' --- a/strings/decimal.c 2009-05-06 12:03:24 +0000 +++ b/strings/decimal.c 2009-08-27 09:48:29 +0000 @@ -306,7 +306,7 @@ int decimal_actual_fraction(decimal_t *f { for (i= DIG_PER_DEC1 - ((frac - 1) % DIG_PER_DEC1); *buf0 % powers10[i++] == 0; - frac--); + frac--) {} } return frac; } @@ -500,7 +500,7 @@ static void digits_bounds(decimal_t *fro stop= (int) ((buf_end - from->buf + 1) * DIG_PER_DEC1); i= 1; } - for (; *buf_end % powers10[i++] == 0; stop--); + for (; *buf_end % powers10[i++] == 0; stop--) {} *end_result= stop; /* index of position after last decimal digit (from 0) */ } @@ -1011,7 +1011,7 @@ static int ull2dec(ulonglong from, decim sanity(to); - for (intg1=1; from >= DIG_BASE; intg1++, from/=DIG_BASE); + for (intg1=1; from >= DIG_BASE; intg1++, from/=DIG_BASE) {} if (unlikely(intg1 > to->len)) { intg1=to->len; === added file 'support-files/ccfilter' --- a/support-files/ccfilter 1970-01-01 00:00:00 +0000 +++ b/support-files/ccfilter 2009-08-27 09:48:29 +0000 @@ -0,0 +1,104 @@ +#! /usr/bin/perl + +# Post-processor for compiler output to filter out warnings matched in +# support-files/compiler_warnings.supp. This makes it easier to check +# that no new warnings are introduced without needing to submit a build +# for Buildbot. +# +# Use by setting CC="ccfilter gcc" CXX="ccfilter gcc" before ./configure. +# +# By default, just filters the output for suppressed warnings. If the +# FAILONWARNING environment variable is set, then instead will fail the +# compile on encountering a non-suppressed warnings. + +use strict; +use warnings; + +my $suppressions; + +open STDOUT_COPY, ">&STDOUT" + or die "Failed to dup stdout: $!]n"; + +my $pid= open(PIPE, '-|'); + +if (!defined($pid)) { + die "Error: Cannot fork(): $!\n"; +} elsif (!$pid) { + # Child. + # actually want to send the STDERR to the parent, not the STDOUT. + # So shuffle things around a bit. + open STDERR, ">&STDOUT" + or die "Child: Failed to dup pipe to parent: $!\n"; + open STDOUT, ">&STDOUT_COPY" + or die "Child: Failed to dup parent stdout: $!\n"; + close STDOUT_COPY; + exec { $ARGV[0] } @ARGV; + die "Child: exec() failed: $!\n"; +} else { + # Parent. + close STDOUT_COPY; + my $cwd= qx(pwd); + chomp($cwd); + while (<PIPE>) { + my $line= $_; + if (/^(.*?):([0-9]+): [Ww]arning: (.*)$/) { + my ($file, $lineno, $msg)= ($1, $2, $3); + $file= "$cwd/$file"; + + next + if check_if_suppressed($file, $lineno, $msg); + die "$line\nGot warning, terminating.\n" + if $ENV{FAILONWARNING}; + print STDERR $line; + next; + } + + print STDERR $line; + } + close(PIPE); +} + +exit 0; + +sub check_if_suppressed { + my ($file, $lineno, $msg)= @_; + load_suppressions() unless defined($suppressions); + for my $s (@$suppressions) { + my ($file_re, $msg_re, $start, $end)= @$s; + if ($file =~ /$file_re/ && + $msg =~ /$msg_re/ && + (!defined($start) || $start <= $lineno) && + (!defined($end) || $end >= $lineno)) { + return 1; + } + } + return undef; +} + +sub load_suppressions { + # First find the suppressions file, might be we need to move up to + # the base directory. + my $path = "support-files/compiler_warnings.supp"; + my $exists; + for (1..10) { + $exists= -f $path; + last if $exists; + $path= '../'. $path; + } + die "Error: Could not find suppression file (out of source dir?).\n" + unless $exists; + + $suppressions= []; + open "F", "<", $path + or die "Error: Could not read suppression file '$path': $!\n"; + while (<F>) { + # Skip comment and empty lines. + next if /^\s*(\#.*)?$/; + die "Invalid syntax in suppression file '$path', line $.:\n$_" + unless /^\s*(.+?)\s*:\s*(.+?)\s*(?:[:]\s*([0-9]+)(?:-([0-9]+))?\s*)?$/; + my ($file_re, $line_re, $start, $end)= ($1, $2, $3, $4); + $end = $start + if defined($start) && !defined($end); + push @$suppressions, [$file_re, $line_re, $start, $end]; + } +} === modified file 'support-files/compiler_warnings.supp' --- a/support-files/compiler_warnings.supp 2008-01-11 17:39:43 +0000 +++ b/support-files/compiler_warnings.supp 2009-08-27 09:48:29 +0000 @@ -25,6 +25,9 @@ sql_yacc.cc : .*switch statement contain pars0grm.tab.c: .*'yyerrorlab' : unreferenced label.* _flex_tmp.c: .*not enough actual parameters for macro 'yywrap'.* pars0lex.l: .*conversion from 'ulint' to 'int', possible loss of data.* +btr/btr0cur\.c: .*value computed is not used.*: 3175-3375 +include/buf0buf\.ic: unused parameter ‘mtr’ +fil/fil0fil\.c: comparison between signed and unsigned : 3100-3199 # # bdb is not critical to keep up to date @@ -41,6 +44,12 @@ db_vrfy.c : .*comparison is always false .*/cmd-line-utils/readline/.* : .* # +# Ignore some warnings in libevent, which is not maintained by us. +# +.*/extra/libevent/.* : .*unused parameter.* +.*/extra/libevent/select\.c : .*comparison between signed and unsigned.* : 270-280 + +# # Ignore all conversion warnings on windows 64 # (Is safe as we are not yet supporting strings >= 2G) # @@ -75,6 +84,17 @@ db_vrfy.c : .*comparison is always false storage/maria/ma_pagecache.c: .*'info_check_pin' defined but not used # +# I think these are due to mix of C and C++. +# +storage/pbxt/ : typedef.*was ignored in this declaration + + +# +# Groff warnings on OpenSUSE. +# +.*/dbug/.*(groff|<standard input>) : .* + +# # Unexplanable (?) stuff # listener.cc : .*conversion from 'SOCKET' to 'int'.*

1 0

[Maria-developers] Updated (by Sanja): Avoid flushing keycache to disk in case of DROPping table (53)
by worklog-noreply＠askmonty.org 27 Aug '09

27 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Avoid flushing keycache to disk in case of DROPping table CREATION DATE..: Thu, 27 Aug 2009, 11:12 SUPERVISOR.....: Monty IMPLEMENTOR....: Sanja COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 53 (http://askmonty.org/worklog/?tid=53) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 24 (hours remain) ORIG. ESTIMATE.: 24 PROGRESS NOTES: -=-=(Sanja - Thu, 27 Aug 2009, 11:15)=-=- High-Level Specification modified. --- /tmp/wklog.53.old.23490 2009-08-27 11:15:24.000000000 +0300 +++ /tmp/wklog.53.new.23490 2009-08-27 11:15:24.000000000 +0300 @@ -1 +1,3 @@ - +To make special handler call 'prepare to drop' which mark table as changed (in +case of server crash just before removing table it force table to be repaired) +then flush keycache without writing on disk. DESCRIPTION: When DROP TABLE executed keycache flushed to disk before removing table which is wasting of resources. So better to have table removed from keycache without writing changes on disk. HIGH-LEVEL SPECIFICATION: To make special handler call 'prepare to drop' which mark table as changed (in case of server crash just before removing table it force table to be repaired) then flush keycache without writing on disk. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Sanja): Avoid flushing keycache to disk in case of DROPping table (53)
by worklog-noreply＠askmonty.org 27 Aug '09

27 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Avoid flushing keycache to disk in case of DROPping table CREATION DATE..: Thu, 27 Aug 2009, 11:12 SUPERVISOR.....: Monty IMPLEMENTOR....: Sanja COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 53 (http://askmonty.org/worklog/?tid=53) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 24 (hours remain) ORIG. ESTIMATE.: 24 PROGRESS NOTES: -=-=(Sanja - Thu, 27 Aug 2009, 11:15)=-=- High-Level Specification modified. --- /tmp/wklog.53.old.23490 2009-08-27 11:15:24.000000000 +0300 +++ /tmp/wklog.53.new.23490 2009-08-27 11:15:24.000000000 +0300 @@ -1 +1,3 @@ - +To make special handler call 'prepare to drop' which mark table as changed (in +case of server crash just before removing table it force table to be repaired) +then flush keycache without writing on disk. DESCRIPTION: When DROP TABLE executed keycache flushed to disk before removing table which is wasting of resources. So better to have table removed from keycache without writing changes on disk. HIGH-LEVEL SPECIFICATION: To make special handler call 'prepare to drop' which mark table as changed (in case of server crash just before removing table it force table to be repaired) then flush keycache without writing on disk. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Sanja): Avoid flushing keycache to disk in case of DROPping table (53)
by worklog-noreply＠askmonty.org 27 Aug '09

27 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Avoid flushing keycache to disk in case of DROPping table CREATION DATE..: Thu, 27 Aug 2009, 11:12 SUPERVISOR.....: Monty IMPLEMENTOR....: Sanja COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 53 (http://askmonty.org/worklog/?tid=53) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 24 (hours remain) ORIG. ESTIMATE.: 24 PROGRESS NOTES: DESCRIPTION: When DROP TABLE executed keycache flushed to disk before removing table which is wasting of resources. So better to have table removed from keycache without writing changes on disk. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Sanja): Avoid flushing keycache to disk in case of DROPping table (53)
by worklog-noreply＠askmonty.org 27 Aug '09

27 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Avoid flushing keycache to disk in case of DROPping table CREATION DATE..: Thu, 27 Aug 2009, 11:12 SUPERVISOR.....: Monty IMPLEMENTOR....: Sanja COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 53 (http://askmonty.org/worklog/?tid=53) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 24 (hours remain) ORIG. ESTIMATE.: 24 PROGRESS NOTES: DESCRIPTION: When DROP TABLE executed keycache flushed to disk before removing table which is wasting of resources. So better to have table removed from keycache without writing changes on disk. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Sanja): Backport key cache to 5.0 (52)
by worklog-noreply＠askmonty.org 27 Aug '09

27 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Backport key cache to 5.0 CREATION DATE..: Thu, 27 Aug 2009, 09:54 SUPERVISOR.....: Monty IMPLEMENTOR....: Monty COPIES TO......: Sanja CATEGORY.......: Server-Sprint TASK ID........: 52 (http://askmonty.org/worklog/?tid=52) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 8 (hours remain) ORIG. ESTIMATE.: 8 PROGRESS NOTES: -=-=(Sanja - Thu, 27 Aug 2009, 09:56)=-=- High-Level Specification modified. --- /tmp/wklog.52.old.20094 2009-08-27 09:56:05.000000000 +0300 +++ /tmp/wklog.52.new.20094 2009-08-27 09:56:05.000000000 +0300 @@ -1 +1 @@ - +Backport keycache to 5.0 -=-=(Sanja - Thu, 27 Aug 2009, 09:55)=-=- Low Level Design modified. --- /tmp/wklog.52.old.20035 2009-08-27 09:55:25.000000000 +0300 +++ /tmp/wklog.52.new.20035 2009-08-27 09:55:25.000000000 +0300 @@ -1 +1 @@ - +Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. DESCRIPTION: Backport keycache to bring bugfixes made in 5.1. HIGH-LEVEL SPECIFICATION: Backport keycache to 5.0 LOW-LEVEL DESIGN: Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Sanja): Backport key cache to 5.0 (52)
by worklog-noreply＠askmonty.org 27 Aug '09

27 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Backport key cache to 5.0 CREATION DATE..: Thu, 27 Aug 2009, 09:54 SUPERVISOR.....: Monty IMPLEMENTOR....: Monty COPIES TO......: Sanja CATEGORY.......: Server-Sprint TASK ID........: 52 (http://askmonty.org/worklog/?tid=52) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 8 (hours remain) ORIG. ESTIMATE.: 8 PROGRESS NOTES: -=-=(Sanja - Thu, 27 Aug 2009, 09:56)=-=- High-Level Specification modified. --- /tmp/wklog.52.old.20094 2009-08-27 09:56:05.000000000 +0300 +++ /tmp/wklog.52.new.20094 2009-08-27 09:56:05.000000000 +0300 @@ -1 +1 @@ - +Backport keycache to 5.0 -=-=(Sanja - Thu, 27 Aug 2009, 09:55)=-=- Low Level Design modified. --- /tmp/wklog.52.old.20035 2009-08-27 09:55:25.000000000 +0300 +++ /tmp/wklog.52.new.20035 2009-08-27 09:55:25.000000000 +0300 @@ -1 +1 @@ - +Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. DESCRIPTION: Backport keycache to bring bugfixes made in 5.1. HIGH-LEVEL SPECIFICATION: Backport keycache to 5.0 LOW-LEVEL DESIGN: Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Sanja): Backport key cache to 5.0 (52)
by worklog-noreply＠askmonty.org 27 Aug '09

27 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Backport key cache to 5.0 CREATION DATE..: Thu, 27 Aug 2009, 09:54 SUPERVISOR.....: Monty IMPLEMENTOR....: Monty COPIES TO......: Sanja CATEGORY.......: Server-Sprint TASK ID........: 52 (http://askmonty.org/worklog/?tid=52) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 8 (hours remain) ORIG. ESTIMATE.: 8 PROGRESS NOTES: -=-=(Sanja - Thu, 27 Aug 2009, 09:55)=-=- Low Level Design modified. --- /tmp/wklog.52.old.20035 2009-08-27 09:55:25.000000000 +0300 +++ /tmp/wklog.52.new.20035 2009-08-27 09:55:25.000000000 +0300 @@ -1 +1 @@ - +Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. DESCRIPTION: Backport keycache to bring bugfixes made in 5.1. LOW-LEVEL DESIGN: Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Sanja): Backport key cache to 5.0 (52)
by worklog-noreply＠askmonty.org 27 Aug '09

27 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Backport key cache to 5.0 CREATION DATE..: Thu, 27 Aug 2009, 09:54 SUPERVISOR.....: Monty IMPLEMENTOR....: Monty COPIES TO......: Sanja CATEGORY.......: Server-Sprint TASK ID........: 52 (http://askmonty.org/worklog/?tid=52) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 8 (hours remain) ORIG. ESTIMATE.: 8 PROGRESS NOTES: -=-=(Sanja - Thu, 27 Aug 2009, 09:55)=-=- Low Level Design modified. --- /tmp/wklog.52.old.20035 2009-08-27 09:55:25.000000000 +0300 +++ /tmp/wklog.52.new.20035 2009-08-27 09:55:25.000000000 +0300 @@ -1 +1 @@ - +Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. DESCRIPTION: Backport keycache to bring bugfixes made in 5.1. LOW-LEVEL DESIGN: Just copy files (mf_keycache.c & key=cache.h) and fix what should be fixed. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Sanja): Backport key cache to 5.0 (52)
by worklog-noreply＠askmonty.org 27 Aug '09

27 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Backport key cache to 5.0 CREATION DATE..: Thu, 27 Aug 2009, 09:54 SUPERVISOR.....: Monty IMPLEMENTOR....: Monty COPIES TO......: Sanja CATEGORY.......: Server-Sprint TASK ID........: 52 (http://askmonty.org/worklog/?tid=52) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 8 (hours remain) ORIG. ESTIMATE.: 8 PROGRESS NOTES: DESCRIPTION: Backport keycache to bring bugfixes made in 5.1. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Sanja): Backport key cache to 5.0 (52)
by worklog-noreply＠askmonty.org 27 Aug '09

27 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Backport key cache to 5.0 CREATION DATE..: Thu, 27 Aug 2009, 09:54 SUPERVISOR.....: Monty IMPLEMENTOR....: Monty COPIES TO......: Sanja CATEGORY.......: Server-Sprint TASK ID........: 52 (http://askmonty.org/worklog/?tid=52) VERSION........: Server-5.0 STATUS.........: Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 8 (hours remain) ORIG. ESTIMATE.: 8 PROGRESS NOTES: DESCRIPTION: Backport keycache to bring bugfixes made in 5.1. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Rev 2748: MWL#17: Table elimination: in file:///home/psergey/dev/maria-5.1-table-elim-r11-gcov/
by Sergey Petrunya 26 Aug '09

26 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r11-gcov/ ------------------------------------------------------------ revno: 2748 revision-id: psergey(a)askmonty.org-20090826210140-y2pba3ucdwnvr956 parent: psergey(a)askmonty.org-20090825150255-46i5gvl2ctd0jcm2 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11-gcov timestamp: Thu 2009-08-27 01:01:40 +0400 message: MWL#17: Table elimination: - Fix a trivial problem when OR-merging two multi-equalities - Amend testsuite to provide full gcov coverage === modified file 'mysql-test/r/table_elim.result' --- a/mysql-test/r/table_elim.result 2009-08-25 10:38:22 +0000 +++ b/mysql-test/r/table_elim.result 2009-08-26 21:01:40 +0000 @@ -352,3 +352,69 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL NULL NULL NULL NULL 4 drop table t1,t2,t3; +# +# Multi-equality tests +# +create table t1 (a int, b int, c int, d int); +insert into t1 values (0,0,0,0),(1,1,1,1),(2,2,2,2),(3,3,3,3); +create table t2 (pk int primary key, b int, c int); +insert into t2 select a,a,a from t1 where a in (1,2); +explain +select t1.* +from t1 left join t2 on t2.pk=t2.c and t2.b=t1.a and t1.a=t1.b and t2.c=t2.b +where t1.d=1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 Using where +explain +select t1.* +from +t1 +left join +t2 +on (t2.pk=t2.c and t2.b=t1.a and t1.a=t1.b and t2.c=t2.b) or +(t2.pk=t2.c and t2.b=t1.a and t1.a=t1.b and t2.c=t2.b) +where t1.d=1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 Using where +#This can't be eliminated: +explain +select t1.* +from +t1 +left join +t2 +on (t2.pk=t2.c and t2.b=t1.a and t2.c=t1.b) or +(t2.pk=t2.c and t1.a=t1.b and t2.c=t1.b) +where t1.d=1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 Using where +1 SIMPLE t2 eq_ref PRIMARY PRIMARY 4 test.t1.b 1 +explain +select t1.* +from +t1 +left join +t2 +on (t2.pk=t2.c and t2.b=t1.a and t2.c=t1.b) or +(t2.pk=t2.c and t2.c=t1.b) +; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 +explain +select t1.* +from t1 left join t2 on t2.pk=3 or t2.pk= 4; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 +1 SIMPLE t2 index PRIMARY PRIMARY 4 NULL 2 Using index +explain +select t1.* +from t1 left join t2 on t2.pk=3 or t2.pk= 3; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 +explain +select t1.* +from t1 left join t2 on (t2.pk=3 and t2.b=3) or (t2.pk= 4 and t2.b=3); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 +1 SIMPLE t2 range PRIMARY PRIMARY 4 NULL 2 Using where +drop table t1, t2; === modified file 'mysql-test/t/table_elim.test' --- a/mysql-test/t/table_elim.test 2009-08-25 10:38:22 +0000 +++ b/mysql-test/t/table_elim.test 2009-08-26 21:01:40 +0000 @@ -278,3 +278,61 @@ drop table t1,t2,t3; +--echo # +--echo # Multi-equality tests +--echo # +create table t1 (a int, b int, c int, d int); +insert into t1 values (0,0,0,0),(1,1,1,1),(2,2,2,2),(3,3,3,3); + +create table t2 (pk int primary key, b int, c int); +insert into t2 select a,a,a from t1 where a in (1,2); + +explain +select t1.* +from t1 left join t2 on t2.pk=t2.c and t2.b=t1.a and t1.a=t1.b and t2.c=t2.b +where t1.d=1; + +explain +select t1.* +from + t1 + left join + t2 + on (t2.pk=t2.c and t2.b=t1.a and t1.a=t1.b and t2.c=t2.b) or + (t2.pk=t2.c and t2.b=t1.a and t1.a=t1.b and t2.c=t2.b) +where t1.d=1; + +--echo #This can't be eliminated: +explain +select t1.* +from + t1 + left join + t2 + on (t2.pk=t2.c and t2.b=t1.a and t2.c=t1.b) or + (t2.pk=t2.c and t1.a=t1.b and t2.c=t1.b) +where t1.d=1; + +explain +select t1.* +from + t1 + left join + t2 + on (t2.pk=t2.c and t2.b=t1.a and t2.c=t1.b) or + (t2.pk=t2.c and t2.c=t1.b) +; + +explain +select t1.* +from t1 left join t2 on t2.pk=3 or t2.pk= 4; + +explain +select t1.* +from t1 left join t2 on t2.pk=3 or t2.pk= 3; + +explain +select t1.* +from t1 left join t2 on (t2.pk=3 and t2.b=3) or (t2.pk= 4 and t2.b=3); + +drop table t1, t2; === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-25 14:59:15 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-26 21:01:40 +0000 @@ -654,7 +654,7 @@ List <Field_value> *fv; if (!(fv= new List<Field_value>)) - break; + break; /* purecov: inspected */ List_iterator<Field_value> it1(*old->mult_equal_fields); List_iterator<Field_value> it2(*new_fields->mult_equal_fields); @@ -664,7 +664,11 @@ while (lfield && rfield) { if (lfield == rfield) + { fv->push_back(lfield); + lfield=it1++; + rfield=it2++; + } else { uint left_ratio= lfield->field->table->tablenr*MAX_FIELDS +

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2721)
by knielsen＠knielsen-hq.org 26 Aug '09

26 Aug '09

#At lp:maria 2721 knielsen(a)knielsen-hq.org 2009-08-26 More compiler warnings fixed. modified: client/mysqlbinlog.cc config/ac-macros/misc.m4 extra/yassl/include/yassl_int.hpp extra/yassl/src/handshake.cpp extra/yassl/src/yassl_imp.cpp extra/yassl/src/yassl_int.cpp extra/yassl/taocrypt/include/modes.hpp extra/yassl/taocrypt/src/asn.cpp mysql-test/valgrind.supp sql/strfunc.cc storage/pbxt/src/datadic_xt.cc storage/pbxt/src/discover_xt.cc strings/decimal.c support-files/compiler_warnings.supp per-file messages: client/mysqlbinlog.cc Fix compiler warnings. config/ac-macros/misc.m4 Fix wrong naming, autoconfig requires _cv_ in cached names. extra/yassl/include/yassl_int.hpp Fix compiler warnings. extra/yassl/src/handshake.cpp Fix compiler warnings. extra/yassl/src/yassl_imp.cpp Fix compiler warnings. extra/yassl/src/yassl_int.cpp Fix compiler warnings. extra/yassl/taocrypt/include/modes.hpp Fix compiler warnings. extra/yassl/taocrypt/src/asn.cpp Fix compiler warnings. mysql-test/valgrind.supp Make dlclose() suppression catch also another possible call path, seen in new test case. sql/strfunc.cc Fix compiler warnings. storage/pbxt/src/datadic_xt.cc Fix compiler warnings. storage/pbxt/src/discover_xt.cc Fix compiler warnings. strings/decimal.c Fix compiler warnings. support-files/compiler_warnings.supp Suppress a few warnings that cannot be meaningfully fixed in source code. === modified file 'client/mysqlbinlog.cc' --- a/client/mysqlbinlog.cc 2009-02-13 16:41:47 +0000 +++ b/client/mysqlbinlog.cc 2009-08-26 12:07:38 +0000 @@ -689,8 +689,8 @@ Exit_status process_event(PRINT_EVENT_IN Format events are not concerned by --offset and such, we always need to read them to be able to process the wanted events. */ - if ((rec_count >= offset) && - ((my_time_t)(ev->when) >= start_datetime) || + if (((rec_count >= offset) && + ((my_time_t)(ev->when) >= start_datetime)) || (ev_type == FORMAT_DESCRIPTION_EVENT)) { if (ev_type != FORMAT_DESCRIPTION_EVENT) === modified file 'config/ac-macros/misc.m4' --- a/config/ac-macros/misc.m4 2008-02-26 17:38:43 +0000 +++ b/config/ac-macros/misc.m4 2009-08-26 12:07:38 +0000 @@ -594,15 +594,15 @@ dnl ------------------------------------ dnl MYSQL_NEEDS_MYSYS_NEW AC_DEFUN([MYSQL_NEEDS_MYSYS_NEW], -[AC_CACHE_CHECK([needs mysys_new helpers], mysql_use_mysys_new, +[AC_CACHE_CHECK([needs mysys_new helpers], mysql_cv_use_mysys_new, [ AC_LANG_PUSH(C++) AC_TRY_LINK([], [ class A { public: int b; }; A *a=new A; a->b=10; delete a; -], mysql_use_mysys_new=no, mysql_use_mysys_new=yes) +], mysql_cv_use_mysys_new=no, mysql_cv_use_mysys_new=yes) AC_LANG_POP(C++) ]) -if test "$mysql_use_mysys_new" = "yes" +if test "$mysql_cv_use_mysys_new" = "yes" then AC_DEFINE([USE_MYSYS_NEW], [1], [Needs to use mysys_new helpers]) fi === modified file 'extra/yassl/include/yassl_int.hpp' --- a/extra/yassl/include/yassl_int.hpp 2008-11-18 16:45:44 +0000 +++ b/extra/yassl/include/yassl_int.hpp 2009-08-26 12:07:38 +0000 @@ -441,7 +441,7 @@ public: const Ciphers& GetCiphers() const; const DH_Parms& GetDH_Parms() const; const Stats& GetStats() const; - const VerifyCallback getVerifyCallback() const; + VerifyCallback getVerifyCallback() const; pem_password_cb GetPasswordCb() const; void* GetUserData() const; bool GetSessionCacheOff() const; === modified file 'extra/yassl/src/handshake.cpp' --- a/extra/yassl/src/handshake.cpp 2008-04-28 16:24:05 +0000 +++ b/extra/yassl/src/handshake.cpp 2009-08-26 12:07:38 +0000 @@ -789,7 +789,7 @@ void processReply(SSL& ssl) { if (ssl.GetError()) return; - if (DoProcessReply(ssl)) + if (DoProcessReply(ssl)) { // didn't complete process if (!ssl.getSocket().IsNonBlocking()) { // keep trying now, blocking ok @@ -799,6 +799,7 @@ void processReply(SSL& ssl) else // user will have try again later, non blocking ssl.SetError(YasslError(SSL_ERROR_WANT_READ)); + } } @@ -872,11 +873,12 @@ void sendServerKeyExchange(SSL& ssl, Buf // send change cipher void sendChangeCipher(SSL& ssl, BufferOutput buffer) { - if (ssl.getSecurity().get_parms().entity_ == server_end) + if (ssl.getSecurity().get_parms().entity_ == server_end) { if (ssl.getSecurity().get_resuming()) ssl.verifyState(clientKeyExchangeComplete); else ssl.verifyState(clientFinishedComplete); + } if (ssl.GetError()) return; ChangeCipherSpec ccs; === modified file 'extra/yassl/src/yassl_imp.cpp' --- a/extra/yassl/src/yassl_imp.cpp 2009-02-13 16:41:47 +0000 +++ b/extra/yassl/src/yassl_imp.cpp 2009-08-26 12:07:38 +0000 @@ -1304,7 +1304,7 @@ void ServerHello::Process(input_buffer&, else ssl.useSecurity().use_connection().sessionID_Set_ = false; - if (ssl.getSecurity().get_resuming()) + if (ssl.getSecurity().get_resuming()) { if (memcmp(session_id_, ssl.getSecurity().get_resume().GetID(), ID_LEN) == 0) { ssl.set_masterSecret(ssl.getSecurity().get_resume().GetSecret()); @@ -1319,6 +1319,7 @@ void ServerHello::Process(input_buffer&, ssl.useSecurity().set_resuming(false); ssl.useLog().Trace("server denied resumption"); } + } if (ssl.CompressionOn() && !compression_method_) ssl.UnSetCompression(); // server isn't supporting yaSSL zlib request === modified file 'extra/yassl/src/yassl_int.cpp' --- a/extra/yassl/src/yassl_int.cpp 2009-02-13 16:41:47 +0000 +++ b/extra/yassl/src/yassl_int.cpp 2009-08-26 12:07:38 +0000 @@ -1833,7 +1833,7 @@ SSL_CTX::GetCA_List() const } -const VerifyCallback SSL_CTX::getVerifyCallback() const +VerifyCallback SSL_CTX::getVerifyCallback() const { return verifyCallback_; } === modified file 'extra/yassl/taocrypt/include/modes.hpp' --- a/extra/yassl/taocrypt/include/modes.hpp 2007-03-23 12:43:09 +0000 +++ b/extra/yassl/taocrypt/include/modes.hpp 2009-08-26 12:07:38 +0000 @@ -95,11 +95,12 @@ inline void Mode_BASE::Process(byte* out { if (mode_ == ECB) ECB_Process(out, in, sz); - else if (mode_ == CBC) + else if (mode_ == CBC) { if (dir_ == ENCRYPTION) CBC_Encrypt(out, in, sz); else CBC_Decrypt(out, in, sz); + } } === modified file 'extra/yassl/taocrypt/src/asn.cpp' --- a/extra/yassl/taocrypt/src/asn.cpp 2009-02-13 16:41:47 +0000 +++ b/extra/yassl/taocrypt/src/asn.cpp 2009-08-26 12:07:38 +0000 @@ -780,11 +780,12 @@ void CertDecoder::GetDate(DateType dt) memcpy(date, source_.get_current(), length); source_.advance(length); - if (!ValidateDate(date, b, dt) && verify_) + if (!ValidateDate(date, b, dt) && verify_) { if (dt == BEFORE) source_.SetError(BEFORE_DATE_E); else source_.SetError(AFTER_DATE_E); + } // save for later use if (dt == BEFORE) { @@ -1061,7 +1062,7 @@ word32 DecodeDSA_Signature(byte* decoded return 0; } word32 rLen = GetLength(source); - if (rLen != 20) + if (rLen != 20) { if (rLen == 21) { // zero at front, eat source.next(); --rLen; @@ -1074,6 +1075,7 @@ word32 DecodeDSA_Signature(byte* decoded source.SetError(DSA_SZ_E); return 0; } + } memcpy(decoded, source.get_buffer() + source.get_index(), rLen); source.advance(rLen); @@ -1083,7 +1085,7 @@ word32 DecodeDSA_Signature(byte* decoded return 0; } word32 sLen = GetLength(source); - if (sLen != 20) + if (sLen != 20) { if (sLen == 21) { source.next(); // zero at front, eat --sLen; @@ -1096,6 +1098,7 @@ word32 DecodeDSA_Signature(byte* decoded source.SetError(DSA_SZ_E); return 0; } + } memcpy(decoded + rLen, source.get_buffer() + source.get_index(), sLen); source.advance(sLen); === modified file 'mysql-test/valgrind.supp' --- a/mysql-test/valgrind.supp 2009-08-05 07:21:37 +0000 +++ b/mysql-test/valgrind.supp 2009-08-26 12:07:38 +0000 @@ -387,6 +387,12 @@ fun:plugin_dl_del(st_mysql_lex_string const*) } +# +# Glibc _dl_close_worker() re-allocates a scope data structure, and frees the +# old one. This isn't a leak, but generates "still reachable" warnings, as +# there is no global destructor code to do a final free() at exit(). +# + { dlclose memory loss from plugin variant 2 Memcheck:Leak @@ -397,7 +403,6 @@ fun:_dlerror_run fun:dlclose fun:_Z15free_plugin_memP12st_plugin_dl - fun:_Z13plugin_dl_delPK19st_mysql_lex_string } { @@ -411,7 +416,6 @@ fun:_dlerror_run fun:dlclose fun:_Z15free_plugin_memP12st_plugin_dl - fun:_Z13plugin_dl_delPK19st_mysql_lex_string } { @@ -424,7 +428,6 @@ obj:/lib*/libdl-*.so fun:dlclose fun:_ZL15free_plugin_memP12st_plugin_dl - fun:_ZL13plugin_dl_delPK19st_mysql_lex_string } { @@ -452,7 +455,6 @@ fun:_dlerror_run fun:dlclose fun:_ZL15free_plugin_memP12st_plugin_dl - fun:_ZL13plugin_dl_delPK19st_mysql_lex_string } { @@ -466,7 +468,6 @@ fun:_dlerror_run fun:dlclose fun:_ZL15free_plugin_memP12st_plugin_dl - fun:_ZL13plugin_dl_delPK19st_mysql_lex_string } { === modified file 'sql/strfunc.cc' --- a/sql/strfunc.cc 2009-04-25 10:05:32 +0000 +++ b/sql/strfunc.cc 2009-08-26 12:07:38 +0000 @@ -148,7 +148,7 @@ static uint parse_name(TYPELIB *lib, con } } else - for (; pos != end && *pos != '=' && *pos !=',' ; pos++); + for (; pos != end && *pos != '=' && *pos !=',' ; pos++) { } uint var_len= (uint) (pos - start); /* Determine which flag it is */ === modified file 'storage/pbxt/src/datadic_xt.cc' --- a/storage/pbxt/src/datadic_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/datadic_xt.cc 2009-08-26 12:07:38 +0000 @@ -2871,5 +2871,5 @@ xtBool XTDDTable::checkCanDrop() { /* no refs or references only itself */ return (dt_trefs == NULL) || - (dt_trefs->tr_next == NULL) && (dt_trefs->tr_fkey->co_table == this); + ((dt_trefs->tr_next == NULL) && (dt_trefs->tr_fkey->co_table == this)); } === modified file 'storage/pbxt/src/discover_xt.cc' --- a/storage/pbxt/src/discover_xt.cc 2009-03-26 12:18:01 +0000 +++ b/storage/pbxt/src/discover_xt.cc 2009-08-26 12:07:38 +0000 @@ -493,8 +493,8 @@ mysql_prepare_create_table(THD *thd, HA_ } /* Don't pack rows in old tables if the user has requested this */ if ((sql_field->flags & BLOB_FLAG) || - sql_field->sql_type == MYSQL_TYPE_VARCHAR && - create_info->row_type != ROW_TYPE_FIXED) + (sql_field->sql_type == MYSQL_TYPE_VARCHAR && + create_info->row_type != ROW_TYPE_FIXED)) (*db_options)|= HA_OPTION_PACK_RECORD; it2.rewind(); } @@ -963,7 +963,7 @@ mysql_prepare_create_table(THD *thd, HA_ sql_field->sql_type == MYSQL_TYPE_VARCHAR || sql_field->pack_flag & FIELDFLAG_BLOB))) { - if (column_nr == 0 && (sql_field->pack_flag & FIELDFLAG_BLOB) || + if ((column_nr == 0 && (sql_field->pack_flag & FIELDFLAG_BLOB)) || sql_field->sql_type == MYSQL_TYPE_VARCHAR) key_info->flags|= HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY; else === modified file 'strings/decimal.c' --- a/strings/decimal.c 2009-05-06 12:03:24 +0000 +++ b/strings/decimal.c 2009-08-26 12:07:38 +0000 @@ -306,7 +306,7 @@ int decimal_actual_fraction(decimal_t *f { for (i= DIG_PER_DEC1 - ((frac - 1) % DIG_PER_DEC1); *buf0 % powers10[i++] == 0; - frac--); + frac--) { } } return frac; } @@ -500,7 +500,7 @@ static void digits_bounds(decimal_t *fro stop= (int) ((buf_end - from->buf + 1) * DIG_PER_DEC1); i= 1; } - for (; *buf_end % powers10[i++] == 0; stop--); + for (; *buf_end % powers10[i++] == 0; stop--) { } *end_result= stop; /* index of position after last decimal digit (from 0) */ } @@ -1011,7 +1011,7 @@ static int ull2dec(ulonglong from, decim sanity(to); - for (intg1=1; from >= DIG_BASE; intg1++, from/=DIG_BASE); + for (intg1=1; from >= DIG_BASE; intg1++, from/=DIG_BASE) { } if (unlikely(intg1 > to->len)) { intg1=to->len; === modified file 'support-files/compiler_warnings.supp' --- a/support-files/compiler_warnings.supp 2009-08-24 13:56:18 +0000 +++ b/support-files/compiler_warnings.supp 2009-08-26 12:07:38 +0000 @@ -84,6 +84,17 @@ db_vrfy.c : .*comparison is always false storage/maria/ma_pagecache.c: .*'info_check_pin' defined but not used # +# I think these are due to mix of C and C++. +# +storage/pbxt/ : typedef.*was ignored in this declaration + + +# +# Groff warnings on OpenSUSE. +# +.*/dbug/.*(groff|<standard input>) : .* + +# # Unexplanable (?) stuff # listener.cc : .*conversion from 'SOCKET' to 'int'.*

2 1

[Maria-developers] Buildbot confused by non-monotonicity of revision numbers in the tree
by Sergey Petrunya 26 Aug '09

26 Aug '09

Hi! Buildbot seems to have a wrong idea about the order of bazaar's revision numbers. It also shows only last five pushes, and these two issues together make it impossible to see the recent status. The problem can be observed in maria-5.1-table-elimination tree: https://internal.askmonty.org/internal/index.php/Image:Buildbot-confused-by… it's building but won't let you see the status. I got the tree into this state as follows: 1. branch off maria-5.1 2. make a number of sequential commits, so that the number of the tip revision grows to be much greater than that of mainline tree. 3. Meanwhile, a few csets were commited into the main maria-5.1 4. Do a maria-5.1 -> maria-5.1-table-elimination merge, using the recommended procedure - make a branch of main - bzr merge --pull from maria-5.1-table-elimination to there. As a result, tip revision in maria-5.1-table-elimination becomes a number which is less than it used to be (all csets made in #2 are collapsed into one Merge cset), and buildbot is confused - it assumes that the revision numbers are monotonically increasing, and that assumption puts my latest revision between some dated revisions. Besides the ordering problem, there might be an identity problem. At the moment the tree has a tip revision number #2723. The problem is that there already was a moment in the past when the tree was in a different state and had a tip revision number 2723. Does this mean that the new test results will overwrite the old ones? BR Sergey -- Sergey Petrunia, Software Developer Monty Program AB, http://askmonty.org Blog: http://s.petrunia.net/blog

2 3

[Maria-developers] Progress (by Knielsen): Extra replication tasks (48)
by worklog-noreply＠askmonty.org 26 Aug '09

26 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Extra replication tasks CREATION DATE..: Sun, 16 Aug 2009, 10:58 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 48 (http://askmonty.org/worklog/?tid=48) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 1 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Wed, 26 Aug 2009, 08:55)=-=- Architecture review, discussion. Worked 1 hour and estimate 0 hours remain (original estimate increased by 1 hour). -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency deleted: 48 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency created: 48 now depends on 46 DESCRIPTION: An umbrella task for replication tasks that are nice to do but are not direct responses for customer requests ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Progress (by Knielsen): Extra replication tasks (48)
by worklog-noreply＠askmonty.org 26 Aug '09

26 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Extra replication tasks CREATION DATE..: Sun, 16 Aug 2009, 10:58 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 48 (http://askmonty.org/worklog/?tid=48) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 1 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Wed, 26 Aug 2009, 08:55)=-=- Architecture review, discussion. Worked 1 hour and estimate 0 hours remain (original estimate increased by 1 hour). -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency deleted: 48 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency created: 48 now depends on 46 DESCRIPTION: An umbrella task for replication tasks that are nice to do but are not direct responses for customer requests ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Progress (by Knielsen): improving mysqlbinlog output and doing rename (39)
by worklog-noreply＠askmonty.org 26 Aug '09

26 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: improving mysqlbinlog output and doing rename CREATION DATE..: Sun, 09 Aug 2009, 12:24 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Client-RawIdeaBin TASK ID........: 39 (http://askmonty.org/worklog/?tid=39) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 33 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Wed, 26 Aug 2009, 08:55)=-=- Architecture review / discussion. Worked 4 hours and estimate 0 hours remain (original estimate increased by 4 hours). -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- Dependency created: 39 now depends on 50 -=-=(Guest - Mon, 17 Aug 2009, 11:17)=-=- Dependency created: 39 now depends on 45 -=-=(Guest - Mon, 17 Aug 2009, 11:17)=-=- Dependency deleted: 39 no longer depends on 45 -=-=(Knielsen - Mon, 17 Aug 2009, 11:13)=-=- Research and architecture review. Worked 12 hours and estimate 0 hours remain (original estimate increased by 12 hours). -=-=(Knielsen - Mon, 17 Aug 2009, 11:13)=-=- Reported zero hours worked. Estimate unchanged. -=-=(Psergey - Sun, 16 Aug 2009, 12:07)=-=- Dependency created: 39 now depends on 49 -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency deleted: 39 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 47 ------------------------------------------------------------ -=-=(View All Progress Notes, 20 total)=-=- http://askmonty.org/worklog/index.pl?tid=39&nolimit=1 DESCRIPTION: A combine task for all replication tasks. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Progress (by Knielsen): improving mysqlbinlog output and doing rename (39)
by worklog-noreply＠askmonty.org 26 Aug '09

26 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: improving mysqlbinlog output and doing rename CREATION DATE..: Sun, 09 Aug 2009, 12:24 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Client-RawIdeaBin TASK ID........: 39 (http://askmonty.org/worklog/?tid=39) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 33 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Wed, 26 Aug 2009, 08:55)=-=- Architecture review / discussion. Worked 4 hours and estimate 0 hours remain (original estimate increased by 4 hours). -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- Dependency created: 39 now depends on 50 -=-=(Guest - Mon, 17 Aug 2009, 11:17)=-=- Dependency created: 39 now depends on 45 -=-=(Guest - Mon, 17 Aug 2009, 11:17)=-=- Dependency deleted: 39 no longer depends on 45 -=-=(Knielsen - Mon, 17 Aug 2009, 11:13)=-=- Research and architecture review. Worked 12 hours and estimate 0 hours remain (original estimate increased by 12 hours). -=-=(Knielsen - Mon, 17 Aug 2009, 11:13)=-=- Reported zero hours worked. Estimate unchanged. -=-=(Psergey - Sun, 16 Aug 2009, 12:07)=-=- Dependency created: 39 now depends on 49 -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency deleted: 39 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 47 ------------------------------------------------------------ -=-=(View All Progress Notes, 20 total)=-=- http://askmonty.org/worklog/index.pl?tid=39&nolimit=1 DESCRIPTION: A combine task for all replication tasks. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Rev 2746: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r11-gcov/
by Sergey Petrunya 25 Aug '09

25 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r11-gcov/ ------------------------------------------------------------ revno: 2746 revision-id: psergey(a)askmonty.org-20090825145915-rxwmfczgqe69r5ju parent: psergey(a)askmonty.org-20090825103822-ldmjinfdh7ymc35r committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11-gcov timestamp: Tue 2009-08-25 17:59:15 +0300 message: MWL#17: Table elimination - Mark gcov deadcode === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-25 10:38:22 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-25 14:59:15 +0000 @@ -197,7 +197,7 @@ public: Value_dep(): bound(FALSE), next(NULL) {} virtual void now_bound(Func_dep_analyzer *fda, Module_dep **bound_modules)=0; - virtual ~Value_dep() {} /* only to shut up compiler warnings */ + virtual ~Value_dep(){} /* purecov: inspected */ /* stop compiler warnings */ bool bound; Value_dep *next; @@ -264,7 +264,7 @@ { public: virtual bool now_bound(Func_dep_analyzer *fda, Value_dep **bound_modules)=0; - virtual ~Module_dep(){} + virtual ~Module_dep(){} /* purecov: inspected */ /* stop compiler warnings */ /* Used to make a linked list of elements that became bound and thus can make elements that depend on them bound, too. @@ -510,7 +510,7 @@ Item_equal *item_equal= (Item_equal*)cond; List<Field_value> *fvl; if (!(fvl= new List<Field_value>)) - break; + break; /* purecov: inspected */ Item_equal_iterator it(*item_equal); Item_field *item; @@ -802,6 +802,7 @@ We've filled the entire equality_mods array. Replace it with a bigger one. We do it somewhat inefficiently but it doesn't matter. */ + /* purecov: begin inspected */ Equality_module *new_arr; if (!(new_arr= new Equality_module[fda->n_equality_mods_alloced *2])) return; @@ -811,6 +812,7 @@ fda->equality_mods= new_arr; *eq_mod= new_arr + (*eq_mod - fda->equality_mods); + /* purecov: end */ } (*eq_mod)->field= field_val; @@ -828,7 +830,7 @@ { Table_value *tbl_dep; if (!(tbl_dep= new Table_value(table))) - return NULL; + return NULL; /* purecov: inspected */ Key_module **key_list= &(tbl_dep->keys); /* Add dependencies for unique keys */ @@ -972,7 +974,7 @@ if (!(buf= current_thd->alloc(bitmap_buffer_size(offset))) || bitmap_init(&fda->expr_deps, (my_bitmap_map*)buf, offset, FALSE)) { - DBUG_RETURN(TRUE); + DBUG_RETURN(TRUE); /* purecov: inspected */ } bitmap_clear_all(&fda->expr_deps); @@ -1077,7 +1079,7 @@ #ifndef DBUG_OFF if (!optimizer_flag(thd, OPTIMIZER_SWITCH_TABLE_ELIMINATION)) - DBUG_VOID_RETURN; + DBUG_VOID_RETURN; /* purecov: inspected */ #endif /* Find the tables that are referred to from WHERE/HAVING */ @@ -1256,7 +1258,7 @@ join->thd->lex->current_select->between_count; if (!(fda.equality_mods= new Equality_module[fda.n_equality_mods_alloced])) - return FALSE; + return FALSE; /* purecov: inspected */ Equality_module* last_eq_mod= fda.equality_mods; @@ -1264,7 +1266,7 @@ if (oj_tbl) { if (!get_table_value(&fda, oj_tbl->table)) - return FALSE; + return FALSE; /* purecov: inspected */ } else { @@ -1274,7 +1276,7 @@ if (tbl->table && (tbl->table->map & dep_tables)) { if (!get_table_value(&fda, tbl->table)) - return FALSE; + return FALSE; /* purecov: inspected */ } } } @@ -1292,7 +1294,7 @@ if (!(fda.outer_join_dep= new Outer_join_module(my_count_bits(dep_tables))) || setup_equality_modules_deps(&fda, &bound_modules)) { - return FALSE; /* OOM, default to non-dependent */ + return FALSE; /* OOM, default to non-dependent */ /* purecov: inspected */ } DBUG_EXECUTE("test", dbug_print_deps(&fda); );

1 0

[Maria-developers] Rev 2745: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r11-gcov/
by Sergey Petrunya 25 Aug '09

25 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r11-gcov/ ------------------------------------------------------------ revno: 2745 revision-id: psergey(a)askmonty.org-20090825103822-ldmjinfdh7ymc35r parent: psergey(a)askmonty.org-20090825092750-mnmatimn7cxm4rl8 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11-gcov timestamp: Tue 2009-08-25 13:38:22 +0300 message: MWL#17: Table elimination - More test coverage - Remove unused code === modified file 'mysql-test/r/table_elim.result' --- a/mysql-test/r/table_elim.result 2009-08-25 09:27:50 +0000 +++ b/mysql-test/r/table_elim.result 2009-08-25 10:38:22 +0000 @@ -314,8 +314,8 @@ insert into t1 values (0,0,0,0),(1,1,1,1),(2,2,2,2),(3,3,3,3); create table t2 (pk int primary key, b int) as select a as pk, a as b from t1 where a in (1,2); -create table t3 (pk int primary key, b int) -as select a as pk, a as b from t1 where a in (1,3); +create table t3 (pk1 int, pk2 int, b int, unique(pk1,pk2)); +insert into t3 select a as pk1, a as pk2, a as b from t1 where a in (1,3); explain select t1.a from t1 left join t2 on t2.pk=t1.a and t2.b<t1.b; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL NULL NULL NULL NULL 4 @@ -344,4 +344,11 @@ explain select t1.a from t1 left join t2 on t2.pk in (t1.a); id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL NULL NULL NULL NULL 4 +explain select t1.a from t1 left join t2 on TRUE; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 +1 SIMPLE t2 index NULL PRIMARY 4 NULL 2 Using index +explain select t1.a from t1 left join t3 on t3.pk1=t1.a and t3.pk2 IS NULL; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 drop table t1,t2,t3; === modified file 'mysql-test/t/table_elim.test' --- a/mysql-test/t/table_elim.test 2009-08-25 09:27:50 +0000 +++ b/mysql-test/t/table_elim.test 2009-08-25 10:38:22 +0000 @@ -274,5 +274,7 @@ explain select t1.a from t1 left join t2 on TRUE; +explain select t1.a from t1 left join t3 on t3.pk1=t1.a and t3.pk2 IS NULL; + drop table t1,t2,t3; === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-25 09:27:50 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-25 10:38:22 +0000 @@ -490,7 +490,7 @@ { Item *tmp=new Item_null; if (tmp) - add_eq_mod(fda, eq_mod, *and_level, cond_func, args[0], args[1]); + add_eq_mod(fda, eq_mod, *and_level, cond_func, args[0], tmp); break; } case Item_func::MULT_EQUAL_FUNC: @@ -828,7 +828,7 @@ { Table_value *tbl_dep; if (!(tbl_dep= new Table_value(table))) - return NULL; /* purecov: inspected */ + return NULL; Key_module **key_list= &(tbl_dep->keys); /* Add dependencies for unique keys */ @@ -853,15 +853,8 @@ static Field_value *get_field_value(Func_dep_analyzer *fda, Field *field) { TABLE *table= field->table; - Table_value *tbl_dep; + Table_value *tbl_dep= fda->table_deps[table->tablenr]; - /* First, get the table*/ - if (!(tbl_dep= fda->table_deps[table->tablenr])) - { - if (!(tbl_dep= get_table_value(fda, table))) - return NULL; - } - /* Try finding the field in field list */ Field_value **pfield= &(tbl_dep->fields); while (*pfield && (*pfield)->field->field_index < field->field_index)

1 0

[Maria-developers] Rev 2744: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r11-gcov/
by Sergey Petrunya 25 Aug '09

25 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r11-gcov/ ------------------------------------------------------------ revno: 2744 revision-id: psergey(a)askmonty.org-20090825092750-mnmatimn7cxm4rl8 parent: psergey(a)askmonty.org-20090824191048-xev0gm1miw6ezz2r committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11-gcov timestamp: Tue 2009-08-25 12:27:50 +0300 message: MWL#17: Table elimination - Add more testcases. - Fix trivial compile failure - Remove handling of "column IN (one_element)". This is converted to equality elsewhere === modified file 'mysql-test/r/table_elim.result' --- a/mysql-test/r/table_elim.result 2009-08-24 08:12:42 +0000 +++ b/mysql-test/r/table_elim.result 2009-08-25 09:27:50 +0000 @@ -307,3 +307,41 @@ select * from t2; a b drop table t1, t2; +# +# Tests with various edge-case ON expressions +# +create table t1 (a int, b int, c int, d int); +insert into t1 values (0,0,0,0),(1,1,1,1),(2,2,2,2),(3,3,3,3); +create table t2 (pk int primary key, b int) +as select a as pk, a as b from t1 where a in (1,2); +create table t3 (pk int primary key, b int) +as select a as pk, a as b from t1 where a in (1,3); +explain select t1.a from t1 left join t2 on t2.pk=t1.a and t2.b<t1.b; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 +explain select t1.a from t1 left join t2 on t2.pk=t1.a or t2.b<t1.b; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 +1 SIMPLE t2 ALL PRIMARY NULL NULL NULL 2 +explain select t1.a from t1 left join t2 on t2.b<t1.b or t2.pk=t1.a; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 +1 SIMPLE t2 ALL PRIMARY NULL NULL NULL 2 +explain select t1.a from t1 left join t2 on t2.pk between 10 and 20; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 +1 SIMPLE t2 index PRIMARY PRIMARY 4 NULL 2 Using index +explain select t1.a from t1 left join t2 on t2.pk between 0.5 and 1.5; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 +1 SIMPLE t2 index PRIMARY PRIMARY 4 NULL 2 Using index +explain select t1.a from t1 left join t2 on t2.pk between 10 and 10; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 +explain select t1.a from t1 left join t2 on t2.pk in (10); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 +explain select t1.a from t1 left join t2 on t2.pk in (t1.a); +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 4 +drop table t1,t2,t3; === modified file 'mysql-test/t/table_elim.test' --- a/mysql-test/t/table_elim.test 2009-08-24 08:12:42 +0000 +++ b/mysql-test/t/table_elim.test 2009-08-25 09:27:50 +0000 @@ -249,3 +249,30 @@ select * from t2; drop table t1, t2; +--echo # +--echo # Tests with various edge-case ON expressions +--echo # +create table t1 (a int, b int, c int, d int); +insert into t1 values (0,0,0,0),(1,1,1,1),(2,2,2,2),(3,3,3,3); + +create table t2 (pk int primary key, b int) + as select a as pk, a as b from t1 where a in (1,2); + +create table t3 (pk1 int, pk2 int, b int, unique(pk1,pk2)); +insert into t3 select a as pk1, a as pk2, a as b from t1 where a in (1,3); + +explain select t1.a from t1 left join t2 on t2.pk=t1.a and t2.b<t1.b; +explain select t1.a from t1 left join t2 on t2.pk=t1.a or t2.b<t1.b; +explain select t1.a from t1 left join t2 on t2.b<t1.b or t2.pk=t1.a; + +explain select t1.a from t1 left join t2 on t2.pk between 10 and 20; +explain select t1.a from t1 left join t2 on t2.pk between 0.5 and 1.5; +explain select t1.a from t1 left join t2 on t2.pk between 10 and 10; + +explain select t1.a from t1 left join t2 on t2.pk in (10); +explain select t1.a from t1 left join t2 on t2.pk in (t1.a); + +explain select t1.a from t1 left join t2 on TRUE; + +drop table t1,t2,t3; + === modified file 'sql/mysqld.cc' --- a/sql/mysqld.cc 2009-08-24 19:10:48 +0000 +++ b/sql/mysqld.cc 2009-08-25 09:27:50 +0000 @@ -393,6 +393,8 @@ "index_merge_intersection=on" #ifndef DBUG_OFF ",table_elimination=on"; +#else + ; #endif static char *mysqld_user, *mysqld_chroot, *log_error_file_ptr; static char *opt_init_slave, *language_ptr, *opt_init_connect; === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-24 19:10:48 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-25 09:27:50 +0000 @@ -467,15 +467,6 @@ Item **args= cond_func->arguments(); switch (cond_func->functype()) { - case Item_func::IN_FUNC: - { - if (cond_func->argument_count() == 2) - { - add_eq_mod(fda, eq_mod, *and_level, cond_func, args[0], args[1]); - add_eq_mod(fda, eq_mod, *and_level, cond_func, args[1], args[0]); - } - break; - } case Item_func::BETWEEN: { Item *fld; @@ -837,7 +828,7 @@ { Table_value *tbl_dep; if (!(tbl_dep= new Table_value(table))) - return NULL; + return NULL; /* purecov: inspected */ Key_module **key_list= &(tbl_dep->keys); /* Add dependencies for unique keys */ @@ -1499,6 +1490,7 @@ #ifndef DBUG_OFF +/* purecov: begin inspected */ static void dbug_print_deps(Func_dep_analyzer *fda) { @@ -1559,6 +1551,7 @@ DBUG_UNLOCK_FILE; DBUG_VOID_RETURN; } +/* purecov: end */ #endif /**

1 0

[Maria-developers] Rev 2743: MWL#17: Table elimination: last fixes in file:///home/psergey/dev/maria-5.1-table-elim-r11/
by Sergey Petrunya 24 Aug '09

24 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r11/ ------------------------------------------------------------ revno: 2743 revision-id: psergey(a)askmonty.org-20090824191048-xev0gm1miw6ezz2r parent: psergey(a)askmonty.org-20090824081242-32o90vv8awk27sut committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Mon 2009-08-24 22:10:48 +0300 message: MWL#17: Table elimination: last fixes - Add an @@optimizer_switch flag for table_elimination for debug build - Better comments === modified file 'mysql-test/t/index_merge_myisam.test' --- a/mysql-test/t/index_merge_myisam.test 2009-03-14 18:58:23 +0000 +++ b/mysql-test/t/index_merge_myisam.test 2009-08-24 19:10:48 +0000 @@ -25,15 +25,19 @@ --echo # we get another @@optimizer_switch user) --echo # +--replace_regex /,table_elimination=on// select @@optimizer_switch; set optimizer_switch='index_merge=off,index_merge_union=off'; +--replace_regex /,table_elimination=on// select @@optimizer_switch; set optimizer_switch='index_merge_union=on'; +--replace_regex /,table_elimination=on// select @@optimizer_switch; set optimizer_switch='default,index_merge_sort_union=off'; +--replace_regex /,table_elimination=on// select @@optimizer_switch; --error ER_WRONG_VALUE_FOR_VAR @@ -71,17 +75,21 @@ set optimizer_switch=default; set optimizer_switch='index_merge=off,index_merge_union=off,default'; +--replace_regex /,table_elimination=on// select @@optimizer_switch; set optimizer_switch=default; # Check setting defaults for global vars +--replace_regex /,table_elimination=on// select @@global.optimizer_switch; set @@global.optimizer_switch=default; +--replace_regex /,table_elimination=on// select @@global.optimizer_switch; --echo # --echo # Check index_merge's @@optimizer_switch flags --echo # +--replace_regex /,table_elimination.on// select @@optimizer_switch; create table t0 (a int); @@ -182,6 +190,7 @@ explain select * from t1 where a=10 and b=10 or c=10; set optimizer_switch=default; +--replace_regex /,table_elimination.on// show variables like 'optimizer_switch'; drop table t0, t1; === modified file 'sql/mysql_priv.h' --- a/sql/mysql_priv.h 2009-04-25 10:05:32 +0000 +++ b/sql/mysql_priv.h 2009-08-24 19:10:48 +0000 @@ -528,14 +528,27 @@ #define OPTIMIZER_SWITCH_INDEX_MERGE_UNION 2 #define OPTIMIZER_SWITCH_INDEX_MERGE_SORT_UNION 4 #define OPTIMIZER_SWITCH_INDEX_MERGE_INTERSECT 8 -#define OPTIMIZER_SWITCH_LAST 16 - + +#ifdef DBUG_OFF +# define OPTIMIZER_SWITCH_LAST 16 +#else +# define OPTIMIZER_SWITCH_TABLE_ELIMINATION 16 +# define OPTIMIZER_SWITCH_LAST 32 +#endif + +#ifdef DBUG_OFF /* The following must be kept in sync with optimizer_switch_str in mysqld.cc */ -#define OPTIMIZER_SWITCH_DEFAULT (OPTIMIZER_SWITCH_INDEX_MERGE | \ - OPTIMIZER_SWITCH_INDEX_MERGE_UNION | \ - OPTIMIZER_SWITCH_INDEX_MERGE_SORT_UNION | \ - OPTIMIZER_SWITCH_INDEX_MERGE_INTERSECT) - +# define OPTIMIZER_SWITCH_DEFAULT (OPTIMIZER_SWITCH_INDEX_MERGE | \ + OPTIMIZER_SWITCH_INDEX_MERGE_UNION | \ + OPTIMIZER_SWITCH_INDEX_MERGE_SORT_UNION | \ + OPTIMIZER_SWITCH_INDEX_MERGE_INTERSECT) +#else +# define OPTIMIZER_SWITCH_DEFAULT (OPTIMIZER_SWITCH_INDEX_MERGE | \ + OPTIMIZER_SWITCH_INDEX_MERGE_UNION | \ + OPTIMIZER_SWITCH_INDEX_MERGE_SORT_UNION | \ + OPTIMIZER_SWITCH_INDEX_MERGE_INTERSECT | \ + OPTIMIZER_SWITCH_TABLE_ELIMINATION) +#endif /* Replication uses 8 bytes to store SQL_MODE in the binary log. The day you === modified file 'sql/mysqld.cc' --- a/sql/mysqld.cc 2009-05-19 09:28:05 +0000 +++ b/sql/mysqld.cc 2009-08-24 19:10:48 +0000 @@ -297,9 +297,14 @@ static const char *optimizer_switch_names[]= { - "index_merge","index_merge_union","index_merge_sort_union", - "index_merge_intersection", "default", NullS + "index_merge","index_merge_union","index_merge_sort_union", + "index_merge_intersection", +#ifndef DBUG_OFF + "table_elimination", +#endif + "default", NullS }; + /* Corresponding defines are named OPTIMIZER_SWITCH_XXX */ static const unsigned int optimizer_switch_names_len[]= { @@ -307,6 +312,9 @@ sizeof("index_merge_union") - 1, sizeof("index_merge_sort_union") - 1, sizeof("index_merge_intersection") - 1, +#ifndef DBUG_OFF + sizeof("table_elimination") - 1, +#endif sizeof("default") - 1 }; TYPELIB optimizer_switch_typelib= { array_elements(optimizer_switch_names)-1,"", @@ -382,7 +390,10 @@ /* Text representation for OPTIMIZER_SWITCH_DEFAULT */ static const char *optimizer_switch_str="index_merge=on,index_merge_union=on," "index_merge_sort_union=on," - "index_merge_intersection=on"; + "index_merge_intersection=on" +#ifndef DBUG_OFF + ",table_elimination=on"; +#endif static char *mysqld_user, *mysqld_chroot, *log_error_file_ptr; static char *opt_init_slave, *language_ptr, *opt_init_connect; static char *default_character_set_name; @@ -6929,8 +6940,11 @@ 0, GET_ULONG, OPT_ARG, MAX_TABLES+1, 0, MAX_TABLES+2, 0, 1, 0}, {"optimizer_switch", OPT_OPTIMIZER_SWITCH, "optimizer_switch=option=val[,option=val...], where option={index_merge, " - "index_merge_union, index_merge_sort_union, index_merge_intersection} and " - "val={on, off, default}.", + "index_merge_union, index_merge_sort_union, index_merge_intersection" +#ifndef DBUG_OFF + ", table_elimination" +#endif + "} and val={on, off, default}.", (uchar**) &optimizer_switch_str, (uchar**) &optimizer_switch_str, 0, GET_STR, REQUIRED_ARG, /*OPTIMIZER_SWITCH_DEFAULT*/0, 0, 0, 0, 0, 0}, === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-24 08:12:42 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-24 19:10:48 +0000 @@ -18,6 +18,7 @@ /* OVERVIEW + ======== This file contains table elimination module. The idea behind table elimination is as follows: suppose we have a left join @@ -36,7 +37,9 @@ null-complemented one) and we don't care about what that record combination is. + MODULE INTERFACE + ================ The module has one entry point - eliminate_tables() function, which one needs to call (once) at some point before the join optimization. @@ -58,23 +61,25 @@ Table elimination is redone on every PS re-execution. + TABLE ELIMINATION ALGORITHM FOR ONE OUTER JOIN + ============================================== - As said above, we can remove inner side of an outer join if it is + As described above, we can remove inner side of an outer join if it is 1. not referred to from any other parts of the query 2. always produces one matching record combination. - We check #1 by doing a recursive descent down the join->join_list while - maintaining a union of used_tables() attribute of all expressions we've seen - "elsewhere". When we encounter an outer join, we check if the bitmap of - tables on its inner side has an intersection with tables that are used + We check #1 by doing a recursive descent down the join->join_list while + maintaining a union of used_tables() attribute of all Item expressions in + other parts of the query. When we encounter an outer join, we check if the + bitmap of tables on its inner side has intersection with tables that are used elsewhere. No intersection means that inner side of the outer join could potentially be eliminated. In order to check #2, one needs to prove that inner side of an outer join - is functionally dependent on the outside. We prove dependency by proving - functional dependency of intermediate objects: + is functionally dependent on the outside. The proof is constructed from + functional dependencies of intermediate objects: - Inner side of outer join is functionally dependent when each of its tables are functionally dependent. (We assume a table is functionally dependent @@ -91,14 +96,15 @@ where expr is functionally-depdendent. - Apparently the above rules can be applied recursively. Also, certain entities - depend on multiple other entities. We model this by a bipartite graph which - has two kinds of nodes: + These relationships are modeled as a bipartite directed graph that has + dependencies as edges and two kinds of nodes: Value nodes: - Table column values (each is a value of tblX.columnY) - - Table nodes (each node represents a table inside an eliminable join nest). - each value is either bound (i.e. functionally dependent) or not. + - Table values (each node represents a table inside the join nest we're + trying to eliminate). + A value has one attribute, it is either bound (i.e. functionally dependent) + or not. Module nodes: - Modules representing tblX.colY=expr equalities. Equality module has @@ -118,11 +124,54 @@ (their expressions are either constant or depend only on tables that are outside of the outer join in question) and performns a breadth-first traversal. If we reach the outer join nest node, it means outer join is - functionally-dependant and can be eliminated. Otherwise it cannot. + functionally-dependant and can be eliminated. Otherwise it cannot be. - HANDLING MULTIPLE NESTED OUTER JOINS - (TODO : explanations why 'local bottom up is sufficient') - + HANDLING MULTIPLE NESTED OUTER JOINS + ==================================== + + Outer joins that are not nested one within another are eliminated + independently. For nested outer joins we have the following considerations: + + 1. ON expressions from children outer joins must be taken into account + + Consider this example: + + SELECT t0.* + FROM + t0 + LEFT JOIN + (t1 LEFT JOIN t2 ON t2.primary_key=t1.col1) + ON + t1.primary_key=t0.col AND t2.col1=t1.col2 + + Here we cannot eliminate the "... LEFT JOIN t2 ON ..." part alone because the + ON clause of top level outer join has references to table t2. + We can eliminate the entire "... LEFT JOIN (t1 LEFT JOIN t2) ON .." part, + but in order to do that, we must look at both ON expressions. + + 2. ON expressions of parent outer joins are useless. + Consider an example: + + SELECT t0.* + FROM + t0 + LEFT JOIN + (t1 LEFT JOIN t2 ON some_expr) + ON + t2.primary_key=t1.col -- (*) + + Here the uppermost ON expression has a clause that gives us functional + dependency of table t2 on t1 and hence could be used to eliminate the + "... LEFT JOIN t2 ON..." part. + However, we would not actually encounter this situation, because before the + table elimination we run simplify_joins(), which, among other things, upon + seeing a functional dependency condition like (*) will convert the outer join + of + + "... LEFT JOIN t2 ON ..." + + into inner join and thus make table elimination not to consider eliminating + table t2. */ class Value_dep; @@ -157,7 +206,7 @@ /* A table field value. There is exactly only one such object for any tblX.fieldY - - the field epends on its table and equalities + - the field depends on its table and equalities - expressions that use the field are its dependencies */ class Field_value : public Value_dep @@ -175,26 +224,23 @@ field_index */ Field_value *next_table_field; - /* - Offset of our part of the bitmap psergey-todo: better comment! + /* + Offset to bits in Func_dep_analyzer::expr_deps */ uint bitmap_offset; - /* - Field became known. Check out - - unique keys we belong to - - expressions that depend on us. - */ void now_bound(Func_dep_analyzer *fda, Module_dep **bound_modules); void signal_from_field_to_exprs(Func_dep_analyzer* fda, Module_dep **bound_modules); }; + /* A table value. There is one Table_value object for every table that can potentially be eliminated. + Dependencies: - table depends on any of its unique keys - - has its fields and embedding outer join as dependency. + - has its fields and embedding outer join as dependency */ class Table_value : public Value_dep { @@ -205,13 +251,13 @@ TABLE *table; Field_value *fields; /* Ordered list of fields that belong to this table */ Key_module *keys; /* Ordered list of Unique keys in this table */ - //Outer_join_module *outer_join_dep; void now_bound(Func_dep_analyzer *fda, Module_dep **bound_modules); }; /* - A 'module'. Module has dependencies + A 'module'. Module has unsatisfied dependencies, number of whose is stored in + unknown_args. Modules also can be linked together in a list. */ class Module_dep : public Sql_alloc @@ -232,10 +278,11 @@ /* This represents either - - "tbl.column= expr" equality dependency, i.e. tbl.column depends on fields + - "tbl.column= expr" equality dependency, i.e. tbl.column depends on fields used in the expression, or - tbl1.col1=tbl2.col2=... multi-equality. */ + class Equality_module : public Module_dep { public: @@ -314,7 +361,11 @@ /* Element for the outer join we're attempting to eliminate */ Outer_join_module *outer_join_dep; - /* Bitmap of how expressions depend on bits */ + /* + Bitmap of how expressions depend on bits. Given a Field_value object, + one can check bitmap_is_set(expr_deps, field_val->bitmap_offset + expr_no) + to see if expression equality_mods[expr_no] depends on the given field. + */ MY_BITMAP expr_deps; }; @@ -453,11 +504,23 @@ } case Item_func::MULT_EQUAL_FUNC: { + /* + The condition is a + + tbl1.field1 = tbl2.field2 = tbl3.field3 [= const_expr] + + multiple-equality. Do two things: + - Collect an ordered List<Field_value> of tblX.colY where tblX is one + of those that we're trying to eliminate. + - rembember if there was a const_expr or tblY.colZ that we can consider + bound. + Store all collected information in a Equality_module object. + */ Item_equal *item_equal= (Item_equal*)cond; List<Field_value> *fvl; if (!(fvl= new List<Field_value>)) break; - + Item_equal_iterator it(*item_equal); Item_field *item; Item *bound_item= item_equal->get_const(); @@ -554,9 +617,9 @@ Equality_module *end, uint and_level) { if (start == new_fields) - return start; // Impossible or + return start; /* (nothing) OR (...) -> (nothing) */ if (new_fields == end) - return start; // No new fields, skip all + return start; /* (...) OR (nothing) -> (nothing) */ Equality_module *first_free=new_fields; @@ -564,28 +627,6 @@ { for (Equality_module *old=start ; old != first_free ; old++) { - /* - Merge multiple-equalities: - A: YES. - (a=b=c) OR (a=b=d) produce "a=b". - - TODO: - sort by (table_ptr, column_index) - then run along the two and produce an intersection - - Q: What about constants? - a=b=3 OR a=b=5 -> a=b= (either 3 or 5) - - a=b OR a=b=5 -> a=b= (any constant) - A: keep the constant iff it is present in both sides and is the same. - - class Multi_equality - { - Item *const_item; - List<...) list; - }; - - */ if (old->field == new_fields->field) { if (!old->field) @@ -616,7 +657,7 @@ } else { - // no single constant/bound item. + /* no single constant/bound item. */ old->expression= NULL; } @@ -975,7 +1016,7 @@ } else { - /* It's a multi-equality*/ + /* It's a multi-equality */ eq_mod->unknown_args= !test(eq_mod->expression); List_iterator<Field_value> it(*eq_mod->mult_equal_fields); Field_value* field_val; @@ -1050,6 +1091,11 @@ if (!join->outer_join) DBUG_VOID_RETURN; +#ifndef DBUG_OFF + if (!optimizer_flag(thd, OPTIMIZER_SWITCH_TABLE_ELIMINATION)) + DBUG_VOID_RETURN; +#endif + /* Find the tables that are referred to from WHERE/HAVING */ used_tables= (join->conds? join->conds->used_tables() : 0) | (join->having? join->having->used_tables() : 0); @@ -1188,7 +1234,7 @@ /* - Check if condition makes the a set of tables functionally-dependent + Check if given condition makes given set of tables functionally-dependent SYNOPSIS check_func_dependency() @@ -1197,8 +1243,8 @@ cond Condition to use DESCRIPTION - Check if condition allows to conclude that the table set is functionally - dependent on everything else. + Check if we can use given condition to infer that the set of given tables + is functionally-dependent on everything else. RETURN TRUE - Yes, functionally dependent @@ -1216,7 +1262,10 @@ Func_dep_analyzer fda(join); - /* Start value */ + /* + Pre-alloc some Equality_module structures. We don't need this to be + guaranteed upper bound. + */ fda.n_equality_mods_alloced= join->thd->lex->current_select->max_equal_elems + (join->thd->lex->current_select->cond_count+1)*2 + @@ -1249,7 +1298,7 @@ fda.usable_tables= dep_tables; /* Analyze the the ON expression and create Equality_module objects and - Field_value objects for their left parts. + Field_value objects for the used fields. */ uint and_level=0; build_eq_mods_for_cond(&fda, &last_eq_mod, &and_level, cond); @@ -1264,14 +1313,14 @@ DBUG_EXECUTE("test", dbug_print_deps(&fda); ); - /* The running wave algorithm itself: */ + /* The forward running wave algorithm: */ Value_dep *bound_values= NULL; while (bound_modules) { for (;bound_modules; bound_modules= bound_modules->next) { if (bound_modules->now_bound(&fda, &bound_values)) - return TRUE; /* Dependent! */ + return TRUE; /* Dependent */ } for (;bound_values; bound_values=bound_values->next) bound_values->now_bound(&fda, &bound_modules); @@ -1280,17 +1329,12 @@ } -/* - Table is known means that - - one more element in outer join nest is known - - all its fields are known -*/ - void Table_value::now_bound(Func_dep_analyzer *fda, Module_dep **bound_modules) { DBUG_PRINT("info", ("table %s is now bound", table->alias)); - + + /* Signal to all fields that they are now bound */ for (Field_value *field_dep= fields; field_dep; field_dep= field_dep->next_table_field) { @@ -1302,6 +1346,7 @@ } } + /* Signal to outer join that one more table is known */ if (fda->outer_join_dep->unknown_args && !--fda->outer_join_dep->unknown_args) { @@ -1318,6 +1363,7 @@ DBUG_PRINT("info", ("field %s.%s is now bound", field->table->alias, field->field_name)); + /* Signal to unique keys and expressions that use this field*/ for (Key_module *key_dep= table->keys; key_dep; key_dep= key_dep->next_table_key) { @@ -1337,8 +1383,8 @@ /* - Walk through expressions that depend on this field and 'notify' them - that this field is no longer unknown. + Walk through expressions that depend on this field and notify them + that this field is now known. */ void Field_value::signal_from_field_to_exprs(Func_dep_analyzer* fda, Module_dep **bound_modules) @@ -1362,16 +1408,16 @@ Value_dep **bound_values) { DBUG_PRINT("info", ("Outer join eliminated")); - return TRUE; /* Signal to finish the process */ + return TRUE; /* Signal out that the search is finished */ } bool Equality_module::now_bound(Func_dep_analyzer *fda, Value_dep **bound_values) { - /* For field=expr and we got to know the expr, so we know the field */ if (mult_equal_fields) { + /* It's a=b=c=... multiple equality. Mark all equality members as known. */ List_iterator<Field_value> it(*mult_equal_fields); Field_value *fv; while ((fv= it++)) @@ -1387,6 +1433,7 @@ } else { + /* It's a fieldX=exprY equality. Mark exprY as known */ if (!field->bound) { /* Mark as bound and add to the list */ @@ -1398,7 +1445,9 @@ return FALSE; } + /* Unique key is known means its table is known */ + bool Key_module::now_bound(Func_dep_analyzer *fda, Value_dep **bound_values) { if (!table->bound)

1 0

[Maria-developers] bzr commit into MariaDB 5.1, with Maria 1.5:maria branch (knielsen:2720)
by knielsen＠knielsen-hq.org 24 Aug '09

24 Aug '09

#At lp:maria 2720 knielsen(a)knielsen-hq.org 2009-08-24 Fix most Compiler warnings seen in buildbot. This includes fixing a bug in option parsing; test case for this bug added. Also add suppressions for a few warnings that cannot be meaningfully fixed by MariaDB developers. added: mysql-test/r/plugin_load2.result mysql-test/t/plugin_load2-master.opt mysql-test/t/plugin_load2.test support-files/ccfilter modified: mysys/my_compress.c mysys/my_getopt.c sql/mysqld.cc storage/xtradb/fil/fil0fil.c storage/xtradb/mtr/mtr0mtr.c storage/xtradb/srv/srv0srv.c storage/xtradb/srv/srv0start.c support-files/compiler_warnings.supp === added file 'mysql-test/r/plugin_load2.result' --- a/mysql-test/r/plugin_load2.result 1970-01-01 00:00:00 +0000 +++ b/mysql-test/r/plugin_load2.result 2009-08-24 13:56:18 +0000 @@ -0,0 +1,2 @@ +SELECT @@global.example_enum_var = 'e2'; +ERROR HY000: Unknown system variable 'example_enum_var' === added file 'mysql-test/t/plugin_load2-master.opt' --- a/mysql-test/t/plugin_load2-master.opt 1970-01-01 00:00:00 +0000 +++ b/mysql-test/t/plugin_load2-master.opt 2009-08-24 13:56:18 +0000 @@ -0,0 +1,3 @@ +$EXAMPLE_PLUGIN_OPT +"--plugin-load=;EXAMPLE=ha_example.so;" +--loose-plugin-example-enum-var=nonexistientvalue === added file 'mysql-test/t/plugin_load2.test' --- a/mysql-test/t/plugin_load2.test 1970-01-01 00:00:00 +0000 +++ b/mysql-test/t/plugin_load2.test 2009-08-24 13:56:18 +0000 @@ -0,0 +1,12 @@ +--source include/have_example_plugin.inc + +# Test for bug in parsing plugin enum option. +# The bug was that the error from parsing a non-existent value was not properly +# handled, so the variable was assigned some arbitrary wrong value. +# +# We test this by passing --loose-plugin-example-enum-var=nonexistientvalue in +# the .opt file of the test case, and check that the variable is not wrongly +# set to a value in this case. + +--error 1193 +SELECT @@global.example_enum_var = 'e2'; === modified file 'mysys/my_compress.c' --- a/mysys/my_compress.c 2009-05-22 12:38:50 +0000 +++ b/mysys/my_compress.c 2009-08-24 13:56:18 +0000 @@ -81,12 +81,13 @@ my_bool my_compress(uchar *packet, size_ This fix is safe, since such memory is only used internally by zlib, so we will not hide any bugs in mysql this way. */ -void *my_az_allocator(void *dummy, unsigned int items, unsigned int size) +void *my_az_allocator(void *dummy __attribute__((unused)), unsigned int items, + unsigned int size) { return my_malloc((size_t)items*(size_t)size, IF_VALGRIND(MY_ZEROFILL, MYF(0))); } -void my_az_free(void *dummy, void *address) +void my_az_free(void *dummy __attribute__((unused)), void *address) { my_free(address, MYF(MY_ALLOW_ZERO_PTR)); } === modified file 'mysys/my_getopt.c' --- a/mysys/my_getopt.c 2009-05-20 15:34:34 +0000 +++ b/mysys/my_getopt.c 2009-08-24 13:56:18 +0000 @@ -603,6 +603,7 @@ static int setval(const struct my_option my_bool set_maximum_value) { int err= 0; + int pos; if (value && argument) { @@ -647,7 +648,9 @@ static int setval(const struct my_option return EXIT_OUT_OF_MEMORY; break; case GET_ENUM: - if (((*(ulong *)result_pos)= find_type(argument, opts->typelib, 2) - 1) < 0) + pos = find_type(argument, opts->typelib, 2) - 1; + (*(ulong *)result_pos)= pos; + if (pos < 0) return EXIT_ARGUMENT_INVALID; break; case GET_SET: === modified file 'sql/mysqld.cc' --- a/sql/mysqld.cc 2009-05-19 09:28:05 +0000 +++ b/sql/mysqld.cc 2009-08-24 13:56:18 +0000 @@ -1002,6 +1002,7 @@ static void close_connections(void) } +#ifdef HAVE_CLOSE_SERVER_SOCK static void close_socket(my_socket sock, const char *info) { DBUG_ENTER("close_socket"); @@ -1021,6 +1022,7 @@ static void close_socket(my_socket sock, } DBUG_VOID_RETURN; } +#endif static void close_server_sock() === modified file 'storage/xtradb/fil/fil0fil.c' --- a/storage/xtradb/fil/fil0fil.c 2009-06-25 01:43:25 +0000 +++ b/storage/xtradb/fil/fil0fil.c 2009-08-24 13:56:18 +0000 @@ -45,7 +45,9 @@ Created 10/25/1995 Heikki Tuuri #include "trx0trx.h" #include "trx0sys.h" #include "pars0pars.h" +#include "row0row.h" #include "row0mysql.h" +#include "que0que.h" /* @@ -3134,7 +3136,7 @@ skip_info: rec_offs_init(offsets_); - fprintf(stderr, "InnoDB: Progress in %:"); + fprintf(stderr, "%s", "InnoDB: Progress in %:"); for (offset = 0; offset < size_bytes; offset += UNIV_PAGE_SIZE) { success = os_file_read(file, page, === modified file 'storage/xtradb/mtr/mtr0mtr.c' --- a/storage/xtradb/mtr/mtr0mtr.c 2009-06-25 01:43:25 +0000 +++ b/storage/xtradb/mtr/mtr0mtr.c 2009-08-24 13:56:18 +0000 @@ -32,6 +32,7 @@ Created 11/26/1995 Heikki Tuuri #include "page0types.h" #include "mtr0log.h" #include "log0log.h" +#include "buf0flu.h" /********************************************************************* Releases the item in the slot given. */ === modified file 'storage/xtradb/srv/srv0srv.c' --- a/storage/xtradb/srv/srv0srv.c 2009-07-06 05:47:15 +0000 +++ b/storage/xtradb/srv/srv0srv.c 2009-08-24 13:56:18 +0000 @@ -1815,7 +1815,6 @@ srv_printf_innodb_monitor( ulint btr_search_sys_subtotal; ulint lock_sys_subtotal; ulint recv_sys_subtotal; - ulint io_counter_subtotal; ulint i; trx_t* trx; === modified file 'storage/xtradb/srv/srv0start.c' --- a/storage/xtradb/srv/srv0start.c 2009-08-03 20:09:53 +0000 +++ b/storage/xtradb/srv/srv0start.c 2009-08-24 13:56:18 +0000 @@ -122,20 +122,6 @@ static char* srv_monitor_file_name; #define SRV_MAX_N_PENDING_SYNC_IOS 100 -/* Avoid warnings when using purify */ - -#ifdef HAVE_valgrind -static int inno_bcmp(register const char *s1, register const char *s2, - register uint len) -{ - while ((len-- != 0) && (*s1++ == *s2++)) - ; - - return(len + 1); -} -#define memcmp(A,B,C) inno_bcmp((A),(B),(C)) -#endif - static char* srv_parse_megabytes( === added file 'support-files/ccfilter' --- a/support-files/ccfilter 1970-01-01 00:00:00 +0000 +++ b/support-files/ccfilter 2009-08-24 13:56:18 +0000 @@ -0,0 +1,104 @@ +#! /usr/bin/perl + +# Post-processor for compiler output to filter out warnings matched in +# support-files/compiler_warnings.supp. This makes it easier to check +# that no new warnings are introduced without needing to submit a build +# for Buildbot. +# +# Use by setting CC="ccfilter gcc" CXX="ccfilter gcc" before ./configure. +# +# By default, just filters the output for suppressed warnings. If the +# FAILONWARNING environment variable is set, then instead will fail the +# compile on encountering a non-suppressed warnings. + +use strict; +use warnings; + +my $suppressions; + +open STDOUT_COPY, ">&STDOUT" + or die "Failed to dup stdout: $!]n"; + +my $pid= open(PIPE, '-|'); + +if (!defined($pid)) { + die "Error: Cannot fork(): $!\n"; +} elsif (!$pid) { + # Child. + # actually want to send the STDERR to the parent, not the STDOUT. + # So shuffle things around a bit. + open STDERR, ">&STDOUT" + or die "Child: Failed to dup pipe to parent: $!\n"; + open STDOUT, ">&STDOUT_COPY" + or die "Child: Failed to dup parent stdout: $!\n"; + close STDOUT_COPY; + exec { $ARGV[0] } @ARGV; + die "Child: exec() failed: $!\n"; +} else { + # Parent. + close STDOUT_COPY; + my $cwd= qx(pwd); + chomp($cwd); + while (<PIPE>) { + my $line= $_; + if (/^(.*?):([0-9]+): [Ww]arning: (.*)$/) { + my ($file, $lineno, $msg)= ($1, $2, $3); + $file= "$cwd/$file"; + + next + if check_if_suppressed($file, $lineno, $msg); + die "$line\nGot warning, terminating.\n" + if $ENV{FAILONWARNING}; + print STDERR $line; + next; + } + + print STDERR $line; + } + close(PIPE); +} + +exit 0; + +sub check_if_suppressed { + my ($file, $lineno, $msg)= @_; + load_suppressions() unless defined($suppressions); + for my $s (@$suppressions) { + my ($file_re, $msg_re, $start, $end)= @$s; + if ($file =~ /$file_re/ && + $msg =~ /$msg_re/ && + (!defined($start) || $start <= $lineno) && + (!defined($end) || $end >= $lineno)) { + return 1; + } + } + return undef; +} + +sub load_suppressions { + # First find the suppressions file, might be we need to move up to + # the base directory. + my $path = "support-files/compiler_warnings.supp"; + my $exists; + for (1..10) { + $exists= -f $path; + last if $exists; + $path= '../'. $path; + } + die "Error: Could not find suppression file (out of source dir?).\n" + unless $exists; + + $suppressions= []; + open "F", "<", $path + or die "Error: Could not read suppression file '$path': $!\n"; + while (<F>) { + # Skip comment and empty lines. + next if /^\s*(\#.*)?$/; + die "Invalid syntax in suppression file '$path', line $.:\n$_" + unless /^\s*(.+?)\s*:\s*(.+?)\s*(?:[:]\s*([0-9]+)(?:-([0-9]+))?\s*)?$/; + my ($file_re, $line_re, $start, $end)= ($1, $2, $3, $4); + $end = $start + if defined($start) && !defined($end); + push @$suppressions, [$file_re, $line_re, $start, $end]; + } +} === modified file 'support-files/compiler_warnings.supp' --- a/support-files/compiler_warnings.supp 2008-01-11 17:39:43 +0000 +++ b/support-files/compiler_warnings.supp 2009-08-24 13:56:18 +0000 @@ -25,6 +25,9 @@ sql_yacc.cc : .*switch statement contain pars0grm.tab.c: .*'yyerrorlab' : unreferenced label.* _flex_tmp.c: .*not enough actual parameters for macro 'yywrap'.* pars0lex.l: .*conversion from 'ulint' to 'int', possible loss of data.* +btr/btr0cur\.c: .*value computed is not used.*: 3175-3375 +include/buf0buf\.ic: unused parameter ‘mtr’ +fil/fil0fil\.c: comparison between signed and unsigned : 3100-3199 # # bdb is not critical to keep up to date @@ -41,6 +44,12 @@ db_vrfy.c : .*comparison is always false .*/cmd-line-utils/readline/.* : .* # +# Ignore some warnings in libevent, which is not maintained by us. +# +.*/extra/libevent/.* : .*unused parameter.* +.*/extra/libevent/select\.c : .*comparison between signed and unsigned.* : 270-280 + +# # Ignore all conversion warnings on windows 64 # (Is safe as we are not yet supporting strings >= 2G) #

1 0

[Maria-developers] Rev 2742: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r11/
by Sergey Petrunya 24 Aug '09

24 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r11/ ------------------------------------------------------------ revno: 2742 revision-id: psergey(a)askmonty.org-20090824081242-32o90vv8awk27sut parent: psergey(a)askmonty.org-20090821133606-2t7hib7wuctqller committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Mon 2009-08-24 10:12:42 +0200 message: MWL#17: Table elimination - Correctly handle the case where we have multi-table DELETE and a table that we're deleting from looks like it could be eliminated. === modified file 'mysql-test/r/table_elim.result' --- a/mysql-test/r/table_elim.result 2009-08-21 07:48:22 +0000 +++ b/mysql-test/r/table_elim.result 2009-08-24 08:12:42 +0000 @@ -278,3 +278,32 @@ 1 SIMPLE t1 index NULL PRIMARY 10 NULL 2 Using index 1 SIMPLE t2 ref a a 3 test.t1.a 2 drop table t1, t2; +# +# check UPDATE/DELETE that look like they could be eliminated +# +create table t1 (a int primary key, b int); +insert into t1 values (1,1),(2,2),(3,3); +create table t2 like t1; +insert into t2 select * from t1; +update t1 left join t2 using (a) set t2.a=t2.a+100; +select * from t1; +a b +1 1 +2 2 +3 3 +select * from t2; +a b +101 1 +102 2 +103 3 +delete from t2; +insert into t2 select * from t1; +delete t2 from t1 left join t2 using (a); +select * from t1; +a b +1 1 +2 2 +3 3 +select * from t2; +a b +drop table t1, t2; === modified file 'mysql-test/t/table_elim.test' --- a/mysql-test/t/table_elim.test 2009-08-21 07:48:22 +0000 +++ b/mysql-test/t/table_elim.test 2009-08-24 08:12:42 +0000 @@ -229,3 +229,23 @@ drop table t1, t2; +--echo # +--echo # check UPDATE/DELETE that look like they could be eliminated +--echo # +create table t1 (a int primary key, b int); +insert into t1 values (1,1),(2,2),(3,3); + +create table t2 like t1; +insert into t2 select * from t1; +update t1 left join t2 using (a) set t2.a=t2.a+100; +select * from t1; +select * from t2; + +delete from t2; +insert into t2 select * from t1; + +delete t2 from t1 left join t2 using (a); +select * from t1; +select * from t2; +drop table t1, t2; + === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-21 13:36:06 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-24 08:12:42 +0000 @@ -1069,16 +1069,26 @@ if (join->select_lex == &thd->lex->select_lex) { - /* Multi-table UPDATE and DELETE: don't eliminate the tables we modify: */ - used_tables |= thd->table_map_for_update; /* Multi-table UPDATE: don't eliminate tables referred from SET statement */ if (thd->lex->sql_command == SQLCOM_UPDATE_MULTI) { + /* Multi-table UPDATE and DELETE: don't eliminate the tables we modify: */ + used_tables |= thd->table_map_for_update; List_iterator<Item> it2(thd->lex->value_list); while ((item= it2++)) used_tables |= item->used_tables(); } + + if (thd->lex->sql_command == SQLCOM_DELETE_MULTI) + { + TABLE_LIST *tbl; + for (tbl= (TABLE_LIST*)thd->lex->auxiliary_table_list.first; + tbl; tbl= tbl->next_local) + { + used_tables |= tbl->table->map; + } + } } table_map all_tables= join->all_tables_map();

1 0

[Maria-developers] Rev 2741: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r11/
by Sergey Petrunya 21 Aug '09

21 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r11/ ------------------------------------------------------------ revno: 2741 revision-id: psergey(a)askmonty.org-20090821133606-2t7hib7wuctqller parent: psergey(a)askmonty.org-20090821074822-6x2gv01r9ltt2bhc committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Fri 2009-08-21 15:36:06 +0200 message: MWL#17: Table elimination - Remove a piece of code that's not needed anymore. === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-21 07:48:22 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-21 13:36:06 +0000 @@ -505,39 +505,6 @@ /* - The only requirement of this function is to order fields in some - deterministic way. -*/ - -int cmp_equal_fields(Item_field *field1, Item_field *field2, void *arg) -{ - int cmp= 0; - bool outer_ref= 0; - if (field2->used_tables() & OUTER_REF_TABLE_BIT) - { - outer_ref= 1; - cmp= -1; - } - if (field2->used_tables() & OUTER_REF_TABLE_BIT) - { - outer_ref= 1; - cmp++; - } - if (outer_ref) - return cmp; - cmp= (int)field2->field->table->tablenr - - (int)field1->field->table->tablenr; - if (cmp) - return cmp < 0 ? -1 : 1; - cmp= (int)field2->field->field_index - - (int)field1->field->field_index; - - return cmp < 0 ? -1 : (cmp ? 1 : 0); - -} - - -/* Perform an OR operation on two (adjacent) Equality_module arrays. SYNOPSIS

1 0

[Maria-developers] Rev 2740: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r11/
by Sergey Petrunya 21 Aug '09

21 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r11/ ------------------------------------------------------------ revno: 2740 revision-id: psergey(a)askmonty.org-20090821074822-6x2gv01r9ltt2bhc parent: psergey(a)askmonty.org-20090820155102-5zvm1m6idmie9mmv committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Fri 2009-08-21 09:48:22 +0200 message: MWL#17: Table elimination - More testcases - Set correct dependencies for non-bound multi-equalities. === modified file 'mysql-test/r/table_elim.result' --- a/mysql-test/r/table_elim.result 2009-08-17 16:07:24 +0000 +++ b/mysql-test/r/table_elim.result 2009-08-21 07:48:22 +0000 @@ -218,6 +218,21 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL NULL NULL NULL NULL 2 1 SIMPLE t2 eq_ref PRIMARY PRIMARY 4 test.t1.col 1 +explain select t1.* +from +t1 left join ( t2 left join t3 on t3.pk=t2.col or t3.pk=t2.col) +on t2.col=t1.col or t2.col=t1.col; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 2 +1 SIMPLE t2 ALL NULL NULL NULL NULL 2 +explain select t1.*, t2.* +from +t1 left join +(t2 left join t3 on t3.pk=t2.col or t3.pk=t2.col) +on t2.pk=t1.col or t2.pk=t1.col; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 2 +1 SIMPLE t2 eq_ref PRIMARY PRIMARY 4 test.t1.col 1 drop table t1, t2, t3; # # Check things that look like functional dependencies but really are not === modified file 'mysql-test/t/table_elim.test' --- a/mysql-test/t/table_elim.test 2009-08-17 16:07:24 +0000 +++ b/mysql-test/t/table_elim.test 2009-08-21 07:48:22 +0000 @@ -175,6 +175,17 @@ explain select t1.*, t2.* from t1 left join (t2 left join t3 on t3.pk=t2.col) on t2.pk=t1.col; +explain select t1.* +from + t1 left join ( t2 left join t3 on t3.pk=t2.col or t3.pk=t2.col) + on t2.col=t1.col or t2.col=t1.col; + +explain select t1.*, t2.* +from + t1 left join + (t2 left join t3 on t3.pk=t2.col or t3.pk=t2.col) + on t2.pk=t1.col or t2.pk=t1.col; + drop table t1, t2, t3; --echo # === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-20 15:51:02 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-21 07:48:22 +0000 @@ -454,8 +454,6 @@ case Item_func::MULT_EQUAL_FUNC: { Item_equal *item_equal= (Item_equal*)cond; - // const item is 'item', field -> NULL. mult_equal_fields <-- an ordered - // list of List<Field_value> *fvl; if (!(fvl= new List<Field_value>)) break; @@ -1001,17 +999,24 @@ deps_recorder.expr_offset= eq_mod - fda->equality_mods; deps_recorder.saw_other_tbl= FALSE; eq_mod->unknown_args= 0; - + + if (eq_mod->field) + { /* Regular tbl.col=expr(tblX1.col1, tblY1.col2, ...) */ - eq_mod->expression->walk(&Item::check_column_usage_processor, FALSE, - (uchar*)&deps_recorder); - - if (!eq_mod->field) + eq_mod->expression->walk(&Item::check_column_usage_processor, FALSE, + (uchar*)&deps_recorder); + } + else { - if (eq_mod->unknown_args) - eq_mod->unknown_args= 1; - if (deps_recorder.saw_other_tbl) - eq_mod->unknown_args= 0; + /* It's a multi-equality*/ + eq_mod->unknown_args= !test(eq_mod->expression); + List_iterator<Field_value> it(*eq_mod->mult_equal_fields); + Field_value* field_val; + while ((field_val= it++)) + { + uint offs= field_val->bitmap_offset + eq_mod - fda->equality_mods; + bitmap_set_bit(&fda->expr_deps, offs); + } } if (!eq_mod->unknown_args)

1 0

[Maria-developers] Rev 2739: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r11/
by Sergey Petrunya 20 Aug '09

20 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r11/ ------------------------------------------------------------ revno: 2739 revision-id: psergey(a)askmonty.org-20090820155102-5zvm1m6idmie9mmv parent: psergey(a)askmonty.org-20090819120659-0ndd6mqxeetgee2n committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Thu 2009-08-20 17:51:02 +0200 message: MWL#17: Table elimination - Multiple-equality handling === modified file 'sql/item_cmpfunc.h' --- a/sql/item_cmpfunc.h 2008-12-12 11:13:11 +0000 +++ b/sql/item_cmpfunc.h 2009-08-20 15:51:02 +0000 @@ -1578,6 +1578,7 @@ uint members(); bool contains(Field *field); Item_field* get_first() { return fields.head(); } + uint n_fields() { return fields.elements; } void merge(Item_equal *item); void update_const(); enum Functype functype() const { return MULT_EQUAL_FUNC; } === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-19 12:06:59 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-20 15:51:02 +0000 @@ -175,7 +175,10 @@ field_index */ Field_value *next_table_field; - uint bitmap_offset; /* Offset of our part of the bitmap */ + /* + Offset of our part of the bitmap psergey-todo: better comment! + */ + uint bitmap_offset; /* Field became known. Check out @@ -214,10 +217,6 @@ class Module_dep : public Sql_alloc { public: - enum { - MODULE_OUTER_JOIN - } type; /* Type of the object */ - virtual bool now_bound(Func_dep_analyzer *fda, Value_dep **bound_modules)=0; virtual ~Module_dep(){} /* @@ -232,8 +231,10 @@ /* - A "tbl.column= expr" equality dependency. tbl.column depends on fields - used in expr. + This represents either + - "tbl.column= expr" equality dependency, i.e. tbl.column depends on fields + used in the expression, or + - tbl1.col1=tbl2.col2=... multi-equality. */ class Equality_module : public Module_dep { @@ -241,6 +242,8 @@ Field_value *field; Item *expression; + List<Field_value> *mult_equal_fields; + /* Used during condition analysis only, similar to KEYUSE::level */ uint level; bool now_bound(Func_dep_analyzer *fda, Value_dep **bound_values); @@ -331,7 +334,7 @@ Item* cond); static -void build_eq_mods_for_cond(Func_dep_analyzer *fda, Equality_module **fdeps, +void build_eq_mods_for_cond(Func_dep_analyzer *fda, Equality_module **eq_mod, uint *and_level, Item *cond); static void add_eq_mod(Func_dep_analyzer *fda, Equality_module **eq_mod, @@ -346,6 +349,9 @@ static Field_value *get_field_value(Func_dep_analyzer *fda, Field *field); static void mark_as_eliminated(JOIN *join, TABLE_LIST *tbl); +static void add_eq_mod2(Func_dep_analyzer *fda, Equality_module **eq_mod, + uint and_level, Field_value *field_val, Item *right, + List<Field_value>* mult_equal_fields); #ifndef DBUG_OFF @@ -360,7 +366,7 @@ SYNOPSIS build_eq_mods_for_cond() fda Table elimination context - fdeps INOUT Put produced equality conditions here + eq_mod INOUT Put produced equality conditions here and_level INOUT AND-level (like in add_key_fields) cond Condition to process @@ -369,34 +375,34 @@ */ static -void build_eq_mods_for_cond(Func_dep_analyzer *fda, Equality_module **fdeps, +void build_eq_mods_for_cond(Func_dep_analyzer *fda, Equality_module **eq_mod, uint *and_level, Item *cond) { if (cond->type() == Item_func::COND_ITEM) { List_iterator_fast<Item> li(*((Item_cond*) cond)->argument_list()); - Equality_module *org_key_fields= *fdeps; + Equality_module *org_key_fields= *eq_mod; /* AND/OR */ if (((Item_cond*) cond)->functype() == Item_func::COND_AND_FUNC) { Item *item; while ((item=li++)) - build_eq_mods_for_cond(fda, fdeps, and_level, item); - for (; org_key_fields != *fdeps ; org_key_fields++) + build_eq_mods_for_cond(fda, eq_mod, and_level, item); + for (; org_key_fields != *eq_mod ; org_key_fields++) org_key_fields->level= *and_level; } else { Item *item; (*and_level)++; - build_eq_mods_for_cond(fda, fdeps, and_level, li++); + build_eq_mods_for_cond(fda, eq_mod, and_level, li++); while ((item=li++)) { - Equality_module *start_key_fields= *fdeps; + Equality_module *start_key_fields= *eq_mod; (*and_level)++; - build_eq_mods_for_cond(fda, fdeps, and_level, item); - *fdeps= merge_func_deps(org_key_fields, start_key_fields, *fdeps, + build_eq_mods_for_cond(fda, eq_mod, and_level, item); + *eq_mod= merge_func_deps(org_key_fields, start_key_fields, *eq_mod, ++(*and_level)); } } @@ -414,8 +420,8 @@ { if (cond_func->argument_count() == 2) { - add_eq_mod(fda, fdeps, *and_level, cond_func, args[0], args[1]); - add_eq_mod(fda, fdeps, *and_level, cond_func, args[1], args[0]); + add_eq_mod(fda, eq_mod, *and_level, cond_func, args[0], args[1]); + add_eq_mod(fda, eq_mod, *and_level, cond_func, args[1], args[0]); } break; } @@ -426,62 +432,72 @@ (fld= args[0]->real_item())->type() == Item::FIELD_ITEM && args[1]->eq(args[2], ((Item_field*)fld)->field->binary())) { - add_eq_mod(fda, fdeps, *and_level, cond_func, args[0], args[1]); - add_eq_mod(fda, fdeps, *and_level, cond_func, args[1], args[0]); + add_eq_mod(fda, eq_mod, *and_level, cond_func, args[0], args[1]); + add_eq_mod(fda, eq_mod, *and_level, cond_func, args[1], args[0]); } break; } case Item_func::EQ_FUNC: case Item_func::EQUAL_FUNC: { - add_eq_mod(fda, fdeps, *and_level, cond_func, args[0], args[1]); - add_eq_mod(fda, fdeps, *and_level, cond_func, args[1], args[0]); + add_eq_mod(fda, eq_mod, *and_level, cond_func, args[0], args[1]); + add_eq_mod(fda, eq_mod, *and_level, cond_func, args[1], args[0]); break; } case Item_func::ISNULL_FUNC: { Item *tmp=new Item_null; if (tmp) - add_eq_mod(fda, fdeps, *and_level, cond_func, args[0], args[1]); + add_eq_mod(fda, eq_mod, *and_level, cond_func, args[0], args[1]); break; } case Item_func::MULT_EQUAL_FUNC: { - Item_equal *item_equal= (Item_equal *) cond; - Item *const_item= item_equal->get_const(); + Item_equal *item_equal= (Item_equal*)cond; + // const item is 'item', field -> NULL. mult_equal_fields <-- an ordered + // list of + List<Field_value> *fvl; + if (!(fvl= new List<Field_value>)) + break; + Item_equal_iterator it(*item_equal); Item_field *item; - if (const_item) - { - /* - For each field field1 from item_equal consider the equality - field1=const_item as a condition allowing an index access of the table - with field1 by the keys value of field1. - */ - while ((item= it++)) - add_eq_mod(fda, fdeps, *and_level, cond_func, item, const_item); - } - else - { - /* - Consider all pairs of different fields included into item_equal. - For each of them (field1, field1) consider the equality - field1=field2 as a condition allowing an index access of the table - with field1 by the keys value of field2. - */ - Item_equal_iterator fi(*item_equal); - while ((item= fi++)) + Item *bound_item= item_equal->get_const(); + while ((item= it++)) + { + if ((item->used_tables() & fda->usable_tables)) { - Field *field= item->field; - Item_field *item2; - while ((item2= it++)) + Field_value *field_val; + if ((field_val= get_field_value(fda, item->field))) { - if (!field->eq(item2->field)) - add_eq_mod(fda, fdeps, *and_level, cond_func, item, item2); + List_iterator<Field_value> it2(*fvl); + Field_value *other_f; + uint field_val_ratio= field_val->field->table->tablenr*MAX_FIELDS + + field_val->field->field_index; + bool added= FALSE; + while ((other_f= it2++)) + { + uint other_f_ratio= other_f->field->table->tablenr*MAX_FIELDS + + other_f->field->field_index; + if (other_f_ratio > field_val_ratio) + { + *(it2.ref())= field_val; + it2.after(other_f); + added= TRUE; + break; + } + } + if (!added) + fvl->push_back(field_val); } - it.rewind(); + } + else + { + if (!bound_item) + bound_item= item; } } + add_eq_mod2(fda, eq_mod, *and_level, NULL, bound_item, fvl); break; } default: @@ -491,6 +507,39 @@ /* + The only requirement of this function is to order fields in some + deterministic way. +*/ + +int cmp_equal_fields(Item_field *field1, Item_field *field2, void *arg) +{ + int cmp= 0; + bool outer_ref= 0; + if (field2->used_tables() & OUTER_REF_TABLE_BIT) + { + outer_ref= 1; + cmp= -1; + } + if (field2->used_tables() & OUTER_REF_TABLE_BIT) + { + outer_ref= 1; + cmp++; + } + if (outer_ref) + return cmp; + cmp= (int)field2->field->table->tablenr - + (int)field1->field->table->tablenr; + if (cmp) + return cmp < 0 ? -1 : 1; + cmp= (int)field2->field->field_index - + (int)field1->field->field_index; + + return cmp < 0 ? -1 : (cmp ? 1 : 0); + +} + + +/* Perform an OR operation on two (adjacent) Equality_module arrays. SYNOPSIS @@ -540,9 +589,9 @@ Equality_module *end, uint and_level) { if (start == new_fields) - return start; // Impossible or + return start; // Impossible or if (new_fields == end) - return start; // No new fields, skip all + return start; // No new fields, skip all Equality_module *first_free=new_fields; @@ -551,40 +600,119 @@ for (Equality_module *old=start ; old != first_free ; old++) { /* - TODO: does it make sense to attempt to merging multiple-equalities? - A: YES. - (a=b=c) OR (a=b=d) produce "a=b". - QQ: - What to use for merging? Trivial N*M algorithm or pre-sort and then - merge ordered sequences? + Merge multiple-equalities: + A: YES. + (a=b=c) OR (a=b=d) produce "a=b". + + TODO: + sort by (table_ptr, column_index) + then run along the two and produce an intersection + + Q: What about constants? + a=b=3 OR a=b=5 -> a=b= (either 3 or 5) + + a=b OR a=b=5 -> a=b= (any constant) + A: keep the constant iff it is present in both sides and is the same. + + class Multi_equality + { + Item *const_item; + List<...) list; + }; + */ if (old->field == new_fields->field) { - if (!new_fields->expression->const_item()) - { - /* - If the value matches, we can use the key reference. - If not, we keep it until we have examined all new values - */ - if (old->expression->eq(new_fields->expression, old->field->field->binary())) - { - old->level= and_level; - } - } - else if (old->expression->eq_by_collation(new_fields->expression, - old->field->field->binary(), - old->field->field->charset())) - { - old->level= and_level; - } - else - { + if (!old->field) + { + /* + OR-ing two multiple equalities. We must compute an intersection of + used fields, and check the constants according to these rules: + + a=b=c=d OR a=c=e=f -> a=c (compute intersection) + a=const1 OR a=b -> (nothing) + a=const1 OR a=const1 -> a=const1 + a=const1 OR a=const2 -> (nothing) + + If we're performing an OR operation over multiple equalities, e.g. + + (a=b=c AND p=q) OR (a=b AND v=z) + + then we'll need to try combining each equality with each. ANDed + equalities are guaranteed to be disjoint, so we'll only get one + hit. + */ + if (old->expression && new_fields->expression && + old->expression->eq_by_collation(new_fields->expression, + old->mult_equal_fields->head()->field->binary(), + old->mult_equal_fields->head()->field->charset())) + { + /* Ok, keep */ + } + else + { + // no single constant/bound item. + old->expression= NULL; + } + + List <Field_value> *fv; + if (!(fv= new List<Field_value>)) + break; + + List_iterator<Field_value> it1(*old->mult_equal_fields); + List_iterator<Field_value> it2(*new_fields->mult_equal_fields); + Field_value *lfield= it1++; + Field_value *rfield= it2++; + // Merge + while (lfield && rfield) + { + if (lfield == rfield) + fv->push_back(lfield); + else + { + uint left_ratio= lfield->field->table->tablenr*MAX_FIELDS + + lfield->field->field_index; + uint right_ratio= rfield->field->table->tablenr*MAX_FIELDS + + rfield->field->field_index; + if (left_ratio < right_ratio) + lfield=it1++; + else + rfield=it2++; + } + } + + if (fv->elements + test(old->expression) > 1) + { + old->mult_equal_fields= fv; + old->level= and_level; + } + } + else if (!new_fields->expression->const_item()) + { + /* + If the value matches, we can use the key reference. + If not, we keep it until we have examined all new values + */ + if (old->expression->eq(new_fields->expression, + old->field->field->binary())) + { + old->level= and_level; + } + } + else if (old->expression->eq_by_collation(new_fields->expression, + old->field->field->binary(), + old->field->field->charset())) + { + old->level= and_level; + } + else + { /* The expressions are different. */ - if (old == --first_free) // If last item - break; - *old= *first_free; // Remove old value - old--; // Retry this value - } + if (old == --first_free) // If last item + break; + *old= *first_free; // Remove old value + old--; // Retry this value + } } } } @@ -596,10 +724,10 @@ for (Equality_module *old=start ; old != first_free ;) { if (old->level != and_level) - { // Not used in all levels + { // Not used in all levels if (old == --first_free) - break; - *old= *first_free; // Remove old value + break; + *old= *first_free; // Remove old value continue; } old++; @@ -659,33 +787,42 @@ return; } } - - if (*eq_mod == fda->equality_mods + fda->n_equality_mods_alloced) - { - /* - We've filled the entire equality_mods array. Replace it with a bigger - one. We do it somewhat inefficiently but it doesn't matter. - */ - Equality_module *new_arr; - if (!(new_arr= new Equality_module[fda->n_equality_mods_alloced *2])) - return; - fda->n_equality_mods_alloced *= 2; - for (int i= 0; i < *eq_mod - fda->equality_mods; i++) - new_arr[i]= fda->equality_mods[i]; - - fda->equality_mods= new_arr; - *eq_mod= new_arr + (*eq_mod - fda->equality_mods); - } - - if (!((*eq_mod)->field= get_field_value(fda, field))) + Field_value *field_val; + if ((field_val= get_field_value(fda, field))) + add_eq_mod2(fda, eq_mod, and_level, field_val, right, NULL); + } +} + + +/* Just add eq_mod w/o any checks */ +static void add_eq_mod2(Func_dep_analyzer *fda, Equality_module **eq_mod, + uint and_level, Field_value *field_val, Item *right, + List<Field_value>* mult_equal_fields) +{ + if (*eq_mod == fda->equality_mods + fda->n_equality_mods_alloced) + { + /* + We've filled the entire equality_mods array. Replace it with a bigger + one. We do it somewhat inefficiently but it doesn't matter. + */ + Equality_module *new_arr; + if (!(new_arr= new Equality_module[fda->n_equality_mods_alloced *2])) return; - (*eq_mod)->expression= right; - (*eq_mod)->level= and_level; - (*eq_mod)++; + fda->n_equality_mods_alloced *= 2; + for (int i= 0; i < *eq_mod - fda->equality_mods; i++) + new_arr[i]= fda->equality_mods[i]; + + fda->equality_mods= new_arr; + *eq_mod= new_arr + (*eq_mod - fda->equality_mods); } + + (*eq_mod)->field= field_val; + (*eq_mod)->expression= right; + (*eq_mod)->level= and_level; + (*eq_mod)->mult_equal_fields= mult_equal_fields; + (*eq_mod)++; } - /* Get a Table_value object for the given table, creating it if necessary. */ @@ -782,11 +919,15 @@ */ fda->equality_mods[expr_offset].unknown_args++; } + else + saw_other_tbl= TRUE; } Func_dep_analyzer *fda; /* Offset of the expression we're processing in the dependency bitmap */ - uint expr_offset; + uint expr_offset; + + bool saw_other_tbl; }; @@ -858,9 +999,21 @@ eq_mod++) { deps_recorder.expr_offset= eq_mod - fda->equality_mods; + deps_recorder.saw_other_tbl= FALSE; eq_mod->unknown_args= 0; + + /* Regular tbl.col=expr(tblX1.col1, tblY1.col2, ...) */ eq_mod->expression->walk(&Item::check_column_usage_processor, FALSE, (uchar*)&deps_recorder); + + if (!eq_mod->field) + { + if (eq_mod->unknown_args) + eq_mod->unknown_args= 1; + if (deps_recorder.saw_other_tbl) + eq_mod->unknown_args= 0; + } + if (!eq_mod->unknown_args) { eq_mod->next= bound_dep; @@ -1235,12 +1388,30 @@ Value_dep **bound_values) { /* For field=expr and we got to know the expr, so we know the field */ - if (!field->bound) - { - /* Mark as bound and add to the list */ - field->bound= TRUE; - field->next= *bound_values; - *bound_values= field; + if (mult_equal_fields) + { + List_iterator<Field_value> it(*mult_equal_fields); + Field_value *fv; + while ((fv= it++)) + { + if (!fv->bound) + { + /* Mark as bound and add to the list */ + fv->bound= TRUE; + fv->next= *bound_values; + *bound_values= fv; + } + } + } + else + { + if (!field->bound) + { + /* Mark as bound and add to the list */ + field->bound= TRUE; + field->next= *bound_values; + *bound_values= field; + } } return FALSE; } @@ -1313,11 +1484,19 @@ String str(buf, sizeof(buf), &my_charset_bin); str.length(0); eq_mod->expression->print(&str, QT_ORDINARY); - fprintf(DBUG_FILE, " equality%d: %s -> %s.%s\n", - eq_mod - fda->equality_mods, - str.c_ptr(), - eq_mod->field->table->table->alias, - eq_mod->field->field->field_name); + if (eq_mod->field) + { + fprintf(DBUG_FILE, " equality%d: %s -> %s.%s\n", + eq_mod - fda->equality_mods, + str.c_ptr(), + eq_mod->field->table->table->alias, + eq_mod->field->field->field_name); + } + else + { + fprintf(DBUG_FILE, " equality%d: multi-equality", + eq_mod - fda->equality_mods); + } } fprintf(DBUG_FILE,"\n");

1 0

[Maria-developers] Rev 2738: Variable/function renames in file:///home/psergey/dev/maria-5.1-table-elim-r11/
by Sergey Petrunya 19 Aug '09

19 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r11/ ------------------------------------------------------------ revno: 2738 revision-id: psergey(a)askmonty.org-20090819120659-0ndd6mqxeetgee2n parent: psergey(a)askmonty.org-20090819101838-55ys0h923olqz4q9 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Wed 2009-08-19 15:06:59 +0300 message: Variable/function renames === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-19 10:18:38 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-19 12:06:59 +0000 @@ -147,7 +147,7 @@ { public: Value_dep(): bound(FALSE), next(NULL) {} - virtual void now_bound(Func_dep_analyzer *te, Module_dep **bound_modules)=0; + virtual void now_bound(Func_dep_analyzer *fda, Module_dep **bound_modules)=0; virtual ~Value_dep() {} /* only to shut up compiler warnings */ bool bound; @@ -182,8 +182,8 @@ - unique keys we belong to - expressions that depend on us. */ - void now_bound(Func_dep_analyzer *te, Module_dep **bound_modules); - void signal_from_field_to_exprs(Func_dep_analyzer* te, + void now_bound(Func_dep_analyzer *fda, Module_dep **bound_modules); + void signal_from_field_to_exprs(Func_dep_analyzer* fda, Module_dep **bound_modules); }; @@ -203,7 +203,7 @@ Field_value *fields; /* Ordered list of fields that belong to this table */ Key_module *keys; /* Ordered list of Unique keys in this table */ //Outer_join_module *outer_join_dep; - void now_bound(Func_dep_analyzer *te, Module_dep **bound_modules); + void now_bound(Func_dep_analyzer *fda, Module_dep **bound_modules); }; @@ -218,7 +218,7 @@ MODULE_OUTER_JOIN } type; /* Type of the object */ - virtual bool now_bound(Func_dep_analyzer *te, Value_dep **bound_modules)=0; + virtual bool now_bound(Func_dep_analyzer *fda, Value_dep **bound_modules)=0; virtual ~Module_dep(){} /* Used to make a linked list of elements that became bound and thus can @@ -243,7 +243,7 @@ /* Used during condition analysis only, similar to KEYUSE::level */ uint level; - bool now_bound(Func_dep_analyzer *te, Value_dep **bound_values); + bool now_bound(Func_dep_analyzer *fda, Value_dep **bound_values); }; @@ -264,7 +264,7 @@ uint keyno; /* Unique keys form a linked list, ordered by keyno */ Key_module *next_table_key; - bool now_bound(Func_dep_analyzer *te, Value_dep **bound_values); + bool now_bound(Func_dep_analyzer *fda, Value_dep **bound_values); }; @@ -280,7 +280,7 @@ { unknown_args= n_children; } - bool now_bound(Func_dep_analyzer *te, Value_dep **bound_values); + bool now_bound(Func_dep_analyzer *fda, Value_dep **bound_values); }; @@ -301,9 +301,9 @@ table_map usable_tables; /* Array of equality dependencies */ - Equality_module *equality_deps; - uint n_equality_deps; /* Number of elements in the array */ - uint n_equality_deps_alloced; + Equality_module *equality_mods; + uint n_equality_mods; /* Number of elements in the array */ + uint n_equality_mods_alloced; /* tablenr -> Table_value* mapping. */ Table_value *table_deps[MAX_KEY]; @@ -331,10 +331,10 @@ Item* cond); static -bool build_eq_deps_for_cond(Func_dep_analyzer *te, Equality_module **fdeps, +void build_eq_mods_for_cond(Func_dep_analyzer *fda, Equality_module **fdeps, uint *and_level, Item *cond); static -bool add_eq_dep(Func_dep_analyzer *te, Equality_module **eq_dep, +void add_eq_mod(Func_dep_analyzer *fda, Equality_module **eq_mod, uint and_level, Item_func *cond, Item *left, Item *right); static @@ -342,14 +342,14 @@ Equality_module *new_fields, Equality_module *end, uint and_level); -static Table_value *get_table_value(Func_dep_analyzer *te, TABLE *table); -static Field_value *get_field_value(Func_dep_analyzer *te, Field *field); +static Table_value *get_table_value(Func_dep_analyzer *fda, TABLE *table); +static Field_value *get_field_value(Func_dep_analyzer *fda, Field *field); static void mark_as_eliminated(JOIN *join, TABLE_LIST *tbl); #ifndef DBUG_OFF -static void dbug_print_deps(Func_dep_analyzer *te); +static void dbug_print_deps(Func_dep_analyzer *fda); #endif /*******************************************************************************************/ @@ -358,17 +358,18 @@ Produce Eq_dep elements for given condition. SYNOPSIS - build_eq_deps_for_cond() - te Table elimination context + build_eq_mods_for_cond() + fda Table elimination context fdeps INOUT Put produced equality conditions here and_level INOUT AND-level (like in add_key_fields) cond Condition to process + DESCRIPTION This function is modeled after add_key_fields() */ static -bool build_eq_deps_for_cond(Func_dep_analyzer *te, Equality_module **fdeps, +void build_eq_mods_for_cond(Func_dep_analyzer *fda, Equality_module **fdeps, uint *and_level, Item *cond) { if (cond->type() == Item_func::COND_ITEM) @@ -381,34 +382,29 @@ { Item *item; while ((item=li++)) - { - if (build_eq_deps_for_cond(te, fdeps, and_level, item)) - return TRUE; - } + build_eq_mods_for_cond(fda, fdeps, and_level, item); for (; org_key_fields != *fdeps ; org_key_fields++) org_key_fields->level= *and_level; } else { + Item *item; (*and_level)++; - if (build_eq_deps_for_cond(te, fdeps, and_level, li++)) - return TRUE; - Item *item; + build_eq_mods_for_cond(fda, fdeps, and_level, li++); while ((item=li++)) { Equality_module *start_key_fields= *fdeps; (*and_level)++; - if (build_eq_deps_for_cond(te, fdeps, and_level, item)) - return TRUE; + build_eq_mods_for_cond(fda, fdeps, and_level, item); *fdeps= merge_func_deps(org_key_fields, start_key_fields, *fdeps, ++(*and_level)); } } - return FALSE; + return; } if (cond->type() != Item::FUNC_ITEM) - return FALSE; + return; Item_func *cond_func= (Item_func*) cond; Item **args= cond_func->arguments(); @@ -418,10 +414,10 @@ { if (cond_func->argument_count() == 2) { - if (add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1]) || - add_eq_dep(te, fdeps, *and_level, cond_func, args[1], args[0])) - return TRUE; + add_eq_mod(fda, fdeps, *and_level, cond_func, args[0], args[1]); + add_eq_mod(fda, fdeps, *and_level, cond_func, args[1], args[0]); } + break; } case Item_func::BETWEEN: { @@ -430,24 +426,23 @@ (fld= args[0]->real_item())->type() == Item::FIELD_ITEM && args[1]->eq(args[2], ((Item_field*)fld)->field->binary())) { - if (add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1]) || - add_eq_dep(te, fdeps, *and_level, cond_func, args[1], args[0])) - return TRUE; + add_eq_mod(fda, fdeps, *and_level, cond_func, args[0], args[1]); + add_eq_mod(fda, fdeps, *and_level, cond_func, args[1], args[0]); } break; } case Item_func::EQ_FUNC: case Item_func::EQUAL_FUNC: { - add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1]); - add_eq_dep(te, fdeps, *and_level, cond_func, args[1], args[0]); + add_eq_mod(fda, fdeps, *and_level, cond_func, args[0], args[1]); + add_eq_mod(fda, fdeps, *and_level, cond_func, args[1], args[0]); break; } case Item_func::ISNULL_FUNC: { Item *tmp=new Item_null; - if (!tmp || add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1])) - return TRUE; + if (tmp) + add_eq_mod(fda, fdeps, *and_level, cond_func, args[0], args[1]); break; } case Item_func::MULT_EQUAL_FUNC: @@ -462,12 +457,9 @@ For each field field1 from item_equal consider the equality field1=const_item as a condition allowing an index access of the table with field1 by the keys value of field1. - */ + */ while ((item= it++)) - { - if (add_eq_dep(te, fdeps, *and_level, cond_func, item, const_item)) - return TRUE; - } + add_eq_mod(fda, fdeps, *and_level, cond_func, item, const_item); } else { @@ -485,10 +477,7 @@ while ((item2= it++)) { if (!field->eq(item2->field)) - { - if (add_eq_dep(te, fdeps, *and_level, cond_func, item, item2)) - return TRUE; - } + add_eq_mod(fda, fdeps, *and_level, cond_func, item, item2); } it.rewind(); } @@ -498,7 +487,6 @@ default: break; } - return FALSE; } @@ -624,8 +612,8 @@ Add an Equality_module element for left=right condition SYNOPSIS - add_eq_dep() - te Table elimination context + add_eq_mod() + fda Table elimination context eq_mod INOUT Store created Equality_module here and increment ptr if you do so and_level AND-level () @@ -645,10 +633,10 @@ */ static -bool add_eq_dep(Func_dep_analyzer *te, Equality_module **eq_mod, +void add_eq_mod(Func_dep_analyzer *fda, Equality_module **eq_mod, uint and_level, Item_func *cond, Item *left, Item *right) { - if ((left->used_tables() & te->usable_tables) && + if ((left->used_tables() & fda->usable_tables) && !(right->used_tables() & RAND_TABLE_BIT) && left->real_item()->type() == Item::FIELD_ITEM) { @@ -658,7 +646,7 @@ if (right->result_type() != STRING_RESULT) { if (field->cmp_type() != right->result_type()) - return FALSE; + return; } else { @@ -668,17 +656,33 @@ */ if (field->cmp_type() == STRING_RESULT && ((Field_str*)field)->charset() != cond->compare_collation()) - return FALSE; + return; } } - - if (!((*eq_mod)->field= get_field_value(te, field))) - return TRUE; + + if (*eq_mod == fda->equality_mods + fda->n_equality_mods_alloced) + { + /* + We've filled the entire equality_mods array. Replace it with a bigger + one. We do it somewhat inefficiently but it doesn't matter. + */ + Equality_module *new_arr; + if (!(new_arr= new Equality_module[fda->n_equality_mods_alloced *2])) + return; + fda->n_equality_mods_alloced *= 2; + for (int i= 0; i < *eq_mod - fda->equality_mods; i++) + new_arr[i]= fda->equality_mods[i]; + + fda->equality_mods= new_arr; + *eq_mod= new_arr + (*eq_mod - fda->equality_mods); + } + + if (!((*eq_mod)->field= get_field_value(fda, field))) + return; (*eq_mod)->expression= right; (*eq_mod)->level= and_level; (*eq_mod)++; } - return FALSE; } @@ -686,7 +690,7 @@ Get a Table_value object for the given table, creating it if necessary. */ -static Table_value *get_table_value(Func_dep_analyzer *te, TABLE *table) +static Table_value *get_table_value(Func_dep_analyzer *fda, TABLE *table) { Table_value *tbl_dep; if (!(tbl_dep= new Table_value(table))) @@ -704,7 +708,7 @@ key_list= &(key_dep->next_table_key); } } - return te->table_deps[table->tablenr]= tbl_dep; + return fda->table_deps[table->tablenr]= tbl_dep; } @@ -712,15 +716,15 @@ Get a Field_value object for the given field, creating it if necessary */ -static Field_value *get_field_value(Func_dep_analyzer *te, Field *field) +static Field_value *get_field_value(Func_dep_analyzer *fda, Field *field) { TABLE *table= field->table; Table_value *tbl_dep; /* First, get the table*/ - if (!(tbl_dep= te->table_deps[table->tablenr])) + if (!(tbl_dep= fda->table_deps[table->tablenr])) { - if (!(tbl_dep= get_table_value(te, table))) + if (!(tbl_dep= get_table_value(fda, table))) return NULL; } @@ -750,13 +754,13 @@ class Field_dependency_recorder : public Field_enumerator { public: - Field_dependency_recorder(Func_dep_analyzer *te_arg): te(te_arg) + Field_dependency_recorder(Func_dep_analyzer *te_arg): fda(te_arg) {} void see_field(Field *field) { Table_value *tbl_dep; - if ((tbl_dep= te->table_deps[field->table->tablenr])) + if ((tbl_dep= fda->table_deps[field->table->tablenr])) { for (Field_value *field_dep= tbl_dep->fields; field_dep; field_dep= field_dep->next_table_field) @@ -764,9 +768,9 @@ if (field->field_index == field_dep->field->field_index) { uint offs= field_dep->bitmap_offset + expr_offset; - if (!bitmap_is_set(&te->expr_deps, offs)) - te->equality_deps[expr_offset].unknown_args++; - bitmap_set_bit(&te->expr_deps, offs); + if (!bitmap_is_set(&fda->expr_deps, offs)) + fda->equality_mods[expr_offset].unknown_args++; + bitmap_set_bit(&fda->expr_deps, offs); return; } } @@ -776,11 +780,11 @@ Bump the dependency anyway, this will signal that this dependency cannot be satisfied. */ - te->equality_deps[expr_offset].unknown_args++; + fda->equality_mods[expr_offset].unknown_args++; } } - Func_dep_analyzer *te; + Func_dep_analyzer *fda; /* Offset of the expression we're processing in the dependency bitmap */ uint expr_offset; }; @@ -791,7 +795,7 @@ SYNOPSIS setup_equality_modules_deps() - te Table elimination context + fda Table elimination context bound_deps_list OUT Start of linked list of elements that were found to be bound (caller will use this to see if that allows to declare further elements bound) @@ -807,7 +811,7 @@ */ static -bool setup_equality_modules_deps(Func_dep_analyzer *te, +bool setup_equality_modules_deps(Func_dep_analyzer *fda, Module_dep **bound_deps_list) { DBUG_ENTER("setup_equality_modules_deps"); @@ -817,8 +821,8 @@ value. */ uint offset= 0; - for (Table_value **tbl_dep=te->table_deps; - tbl_dep < te->table_deps + MAX_TABLES; + for (Table_value **tbl_dep=fda->table_deps; + tbl_dep < fda->table_deps + MAX_TABLES; tbl_dep++) { if (*tbl_dep) @@ -828,39 +832,39 @@ field_dep= field_dep->next_table_field) { field_dep->bitmap_offset= offset; - offset += te->n_equality_deps; + offset += fda->n_equality_mods; } } } void *buf; if (!(buf= current_thd->alloc(bitmap_buffer_size(offset))) || - bitmap_init(&te->expr_deps, (my_bitmap_map*)buf, offset, FALSE)) + bitmap_init(&fda->expr_deps, (my_bitmap_map*)buf, offset, FALSE)) { DBUG_RETURN(TRUE); } - bitmap_clear_all(&te->expr_deps); + bitmap_clear_all(&fda->expr_deps); /* - Analyze all "field=expr" dependencies, and have te->expr_deps encode + Analyze all "field=expr" dependencies, and have fda->expr_deps encode dependencies of expressions from fields. Also collect a linked list of equalities that are bound. */ Module_dep *bound_dep= NULL; - Field_dependency_recorder deps_recorder(te); - for (Equality_module *eq_dep= te->equality_deps; - eq_dep < te->equality_deps + te->n_equality_deps; - eq_dep++) + Field_dependency_recorder deps_recorder(fda); + for (Equality_module *eq_mod= fda->equality_mods; + eq_mod < fda->equality_mods + fda->n_equality_mods; + eq_mod++) { - deps_recorder.expr_offset= eq_dep - te->equality_deps; - eq_dep->unknown_args= 0; - eq_dep->expression->walk(&Item::check_column_usage_processor, FALSE, + deps_recorder.expr_offset= eq_mod - fda->equality_mods; + eq_mod->unknown_args= 0; + eq_mod->expression->walk(&Item::check_column_usage_processor, FALSE, (uchar*)&deps_recorder); - if (!eq_dep->unknown_args) + if (!eq_mod->unknown_args) { - eq_dep->next= bound_dep; - bound_dep= eq_dep; + eq_mod->next= bound_dep; + bound_dep= eq_mod; } } *bound_deps_list= bound_dep; @@ -968,7 +972,7 @@ SYNOPSIS eliminate_tables_for_list() - te Table elimination context + fda Table elimination context join_list Join list to work on list_tables Bitmap of tables embedded in the join_list. on_expr ON expression, if the join list is the inner side @@ -1053,7 +1057,7 @@ SYNOPSIS check_func_dependency() - te Table elimination context + fda Table elimination context tables Set of tables we want to be functionally dependent cond Condition to use @@ -1073,24 +1077,25 @@ TABLE_LIST *oj_tbl, Item* cond) { - uint and_level=0; Module_dep *bound_modules; - //psergey-todo: move allocs to somewhere else. - Func_dep_analyzer pte(join); - Func_dep_analyzer *te= &pte; - uint m= max(join->thd->lex->current_select->max_equal_elems,1); - uint max_elems= ((join->thd->lex->current_select->cond_count+1)*2 + - join->thd->lex->current_select->between_count)*m + 1 + 10; - if (!(te->equality_deps= new Equality_module[max_elems])) + Func_dep_analyzer fda(join); + + /* Start value */ + fda.n_equality_mods_alloced= + join->thd->lex->current_select->max_equal_elems + + (join->thd->lex->current_select->cond_count+1)*2 + + join->thd->lex->current_select->between_count; + + if (!(fda.equality_mods= new Equality_module[fda.n_equality_mods_alloced])) return FALSE; - Equality_module* eq_dep= te->equality_deps; + Equality_module* last_eq_mod= fda.equality_mods; /* Create Table_value objects for all tables we're trying to eliminate */ if (oj_tbl) { - if (!get_table_value(te, oj_tbl->table)) + if (!get_table_value(&fda, oj_tbl->table)) return FALSE; } else @@ -1100,30 +1105,29 @@ { if (tbl->table && (tbl->table->map & dep_tables)) { - if (!get_table_value(te, tbl->table)) + if (!get_table_value(&fda, tbl->table)) return FALSE; } } } - te->usable_tables= dep_tables; + fda.usable_tables= dep_tables; /* Analyze the the ON expression and create Equality_module objects and Field_value objects for their left parts. */ - if (build_eq_deps_for_cond(te, &eq_dep, &and_level, cond) || - eq_dep == te->equality_deps) - return FALSE; - - te->n_equality_deps= eq_dep - te->equality_deps; + uint and_level=0; + build_eq_mods_for_cond(&fda, &last_eq_mod, &and_level, cond); + if (!(fda.n_equality_mods= last_eq_mod - fda.equality_mods)) + return FALSE; /* No useful conditions */ - if (!(te->outer_join_dep= new Outer_join_module(my_count_bits(dep_tables))) || - setup_equality_modules_deps(te, &bound_modules)) + if (!(fda.outer_join_dep= new Outer_join_module(my_count_bits(dep_tables))) || + setup_equality_modules_deps(&fda, &bound_modules)) { return FALSE; /* OOM, default to non-dependent */ } - DBUG_EXECUTE("test", dbug_print_deps(te); ); + DBUG_EXECUTE("test", dbug_print_deps(&fda); ); /* The running wave algorithm itself: */ Value_dep *bound_values= NULL; @@ -1131,11 +1135,11 @@ { for (;bound_modules; bound_modules= bound_modules->next) { - if (bound_modules->now_bound(te, &bound_values)) + if (bound_modules->now_bound(&fda, &bound_values)) return TRUE; /* Dependent! */ } for (;bound_values; bound_values=bound_values->next) - bound_values->now_bound(te, &bound_modules); + bound_values->now_bound(&fda, &bound_modules); } return FALSE; /* Not dependent */ } @@ -1147,7 +1151,7 @@ - all its fields are known */ -void Table_value::now_bound(Func_dep_analyzer *te, +void Table_value::now_bound(Func_dep_analyzer *fda, Module_dep **bound_modules) { DBUG_PRINT("info", ("table %s is now bound", table->alias)); @@ -1159,21 +1163,21 @@ { /* Mark as bound and add to the list */ field_dep->bound= TRUE; - field_dep->signal_from_field_to_exprs(te, bound_modules); + field_dep->signal_from_field_to_exprs(fda, bound_modules); } } - if (te->outer_join_dep->unknown_args && - !--te->outer_join_dep->unknown_args) + if (fda->outer_join_dep->unknown_args && + !--fda->outer_join_dep->unknown_args) { /* Mark as bound and add to the list */ - te->outer_join_dep->next= *bound_modules; - *bound_modules= te->outer_join_dep; + fda->outer_join_dep->next= *bound_modules; + *bound_modules= fda->outer_join_dep; } } -void Field_value::now_bound(Func_dep_analyzer *te, +void Field_value::now_bound(Func_dep_analyzer *fda, Module_dep **bound_modules) { DBUG_PRINT("info", ("field %s.%s is now bound", field->table->alias, @@ -1193,7 +1197,7 @@ *bound_modules= key_dep; } } - signal_from_field_to_exprs(te, bound_modules); + signal_from_field_to_exprs(fda, bound_modules); } @@ -1201,25 +1205,25 @@ Walk through expressions that depend on this field and 'notify' them that this field is no longer unknown. */ -void Field_value::signal_from_field_to_exprs(Func_dep_analyzer* te, +void Field_value::signal_from_field_to_exprs(Func_dep_analyzer* fda, Module_dep **bound_modules) { - for (uint i=0; i < te->n_equality_deps; i++) + for (uint i=0; i < fda->n_equality_mods; i++) { - if (bitmap_is_set(&te->expr_deps, bitmap_offset + i) && - te->equality_deps[i].unknown_args && - !--te->equality_deps[i].unknown_args) + if (bitmap_is_set(&fda->expr_deps, bitmap_offset + i) && + fda->equality_mods[i].unknown_args && + !--fda->equality_mods[i].unknown_args) { /* Mark as bound and add to the list */ - Equality_module* eq_dep= &te->equality_deps[i]; - eq_dep->next= *bound_modules; - *bound_modules= eq_dep; + Equality_module* eq_mod= &fda->equality_mods[i]; + eq_mod->next= *bound_modules; + *bound_modules= eq_mod; } } } -bool Outer_join_module::now_bound(Func_dep_analyzer *te, +bool Outer_join_module::now_bound(Func_dep_analyzer *fda, Value_dep **bound_values) { DBUG_PRINT("info", ("Outer join eliminated")); @@ -1227,7 +1231,7 @@ } -bool Equality_module::now_bound(Func_dep_analyzer *te, +bool Equality_module::now_bound(Func_dep_analyzer *fda, Value_dep **bound_values) { /* For field=expr and we got to know the expr, so we know the field */ @@ -1242,7 +1246,7 @@ } /* Unique key is known means its table is known */ -bool Key_module::now_bound(Func_dep_analyzer *te, Value_dep **bound_values) +bool Key_module::now_bound(Func_dep_analyzer *fda, Value_dep **bound_values) { if (!table->bound) { @@ -1294,7 +1298,7 @@ #ifndef DBUG_OFF static -void dbug_print_deps(Func_dep_analyzer *te) +void dbug_print_deps(Func_dep_analyzer *fda) { DBUG_ENTER("dbug_print_deps"); DBUG_LOCK_FILE; @@ -1302,18 +1306,18 @@ fprintf(DBUG_FILE,"deps {\n"); /* Start with printing equalities */ - for (Equality_module *eq_dep= te->equality_deps; - eq_dep != te->equality_deps + te->n_equality_deps; eq_dep++) + for (Equality_module *eq_mod= fda->equality_mods; + eq_mod != fda->equality_mods + fda->n_equality_mods; eq_mod++) { char buf[128]; String str(buf, sizeof(buf), &my_charset_bin); str.length(0); - eq_dep->expression->print(&str, QT_ORDINARY); + eq_mod->expression->print(&str, QT_ORDINARY); fprintf(DBUG_FILE, " equality%d: %s -> %s.%s\n", - eq_dep - te->equality_deps, + eq_mod - fda->equality_mods, str.c_ptr(), - eq_dep->field->table->table->alias, - eq_dep->field->field->field_name); + eq_mod->field->table->table->alias, + eq_mod->field->field->field_name); } fprintf(DBUG_FILE,"\n"); @@ -1321,7 +1325,7 @@ for (uint i=0; i < MAX_TABLES; i++) { Table_value *table_dep; - if ((table_dep= te->table_deps[i])) + if ((table_dep= fda->table_deps[i])) { /* Print table */ fprintf(DBUG_FILE, " table %s\n", table_dep->table->alias); @@ -1332,9 +1336,9 @@ fprintf(DBUG_FILE, " field %s.%s ->", table_dep->table->alias, field_dep->field->field_name); uint ofs= field_dep->bitmap_offset; - for (uint bit= ofs; bit < ofs + te->n_equality_deps; bit++) + for (uint bit= ofs; bit < ofs + fda->n_equality_mods; bit++) { - if (bitmap_is_set(&te->expr_deps, bit)) + if (bitmap_is_set(&fda->expr_deps, bit)) fprintf(DBUG_FILE, " equality%d ", bit - ofs); } fprintf(DBUG_FILE, "\n");

1 0

[Maria-developers] Rev 2737: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r11/
by Sergey Petrunya 19 Aug '09

19 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r11/ ------------------------------------------------------------ revno: 2737 revision-id: psergey(a)askmonty.org-20090819101838-55ys0h923olqz4q9 parent: psergey(a)askmonty.org-20090818221948-f2mg0szfqrve06f9 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Wed 2009-08-19 13:18:38 +0300 message: MWL#17: Table elimination - Use Table_elimination only for functional dependency checking for individual objects and rename it to Func_dep_analyzer === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-18 22:19:48 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-19 10:18:38 +0000 @@ -135,7 +135,7 @@ class Outer_join_module; class Key_module; -class Table_elimination; +class Func_dep_analyzer; /* @@ -147,7 +147,7 @@ { public: Value_dep(): bound(FALSE), next(NULL) {} - virtual void now_bound(Table_elimination *te, Module_dep **bound_modules)=0; + virtual void now_bound(Func_dep_analyzer *te, Module_dep **bound_modules)=0; virtual ~Value_dep() {} /* only to shut up compiler warnings */ bool bound; @@ -182,8 +182,8 @@ - unique keys we belong to - expressions that depend on us. */ - void now_bound(Table_elimination *te, Module_dep **bound_modules); - void signal_from_field_to_exprs(Table_elimination* te, + void now_bound(Func_dep_analyzer *te, Module_dep **bound_modules); + void signal_from_field_to_exprs(Func_dep_analyzer* te, Module_dep **bound_modules); }; @@ -203,7 +203,7 @@ Field_value *fields; /* Ordered list of fields that belong to this table */ Key_module *keys; /* Ordered list of Unique keys in this table */ //Outer_join_module *outer_join_dep; - void now_bound(Table_elimination *te, Module_dep **bound_modules); + void now_bound(Func_dep_analyzer *te, Module_dep **bound_modules); }; @@ -218,7 +218,7 @@ MODULE_OUTER_JOIN } type; /* Type of the object */ - virtual bool now_bound(Table_elimination *te, Value_dep **bound_modules)=0; + virtual bool now_bound(Func_dep_analyzer *te, Value_dep **bound_modules)=0; virtual ~Module_dep(){} /* Used to make a linked list of elements that became bound and thus can @@ -243,7 +243,7 @@ /* Used during condition analysis only, similar to KEYUSE::level */ uint level; - bool now_bound(Table_elimination *te, Value_dep **bound_values); + bool now_bound(Func_dep_analyzer *te, Value_dep **bound_values); }; @@ -264,7 +264,7 @@ uint keyno; /* Unique keys form a linked list, ordered by keyno */ Key_module *next_table_key; - bool now_bound(Table_elimination *te, Value_dep **bound_values); + bool now_bound(Func_dep_analyzer *te, Value_dep **bound_values); }; @@ -280,26 +280,30 @@ { unknown_args= n_children; } - bool now_bound(Table_elimination *te, Value_dep **bound_values); + bool now_bound(Func_dep_analyzer *te, Value_dep **bound_values); }; /* - Table elimination context + Functional dependency analyzer context */ -class Table_elimination +class Func_dep_analyzer { public: - Table_elimination(JOIN *join_arg) : join(join_arg) + Func_dep_analyzer(JOIN *join_arg) : join(join_arg) { bzero(table_deps, sizeof(table_deps)); } JOIN *join; /* Join we're working on */ + + /* Tables that we're looking at eliminating */ + table_map usable_tables; /* Array of equality dependencies */ Equality_module *equality_deps; uint n_equality_deps; /* Number of elements in the array */ + uint n_equality_deps_alloced; /* tablenr -> Table_value* mapping. */ Table_value *table_deps[MAX_KEY]; @@ -327,27 +331,25 @@ Item* cond); static -bool build_eq_deps_for_cond(Table_elimination *te, Equality_module **fdeps, - uint *and_level, Item *cond, - table_map usable_tables); +bool build_eq_deps_for_cond(Func_dep_analyzer *te, Equality_module **fdeps, + uint *and_level, Item *cond); static -bool add_eq_dep(Table_elimination *te, Equality_module **eq_dep, +bool add_eq_dep(Func_dep_analyzer *te, Equality_module **eq_dep, uint and_level, - Item_func *cond, Item *left, Item *right, - table_map usable_tables); + Item_func *cond, Item *left, Item *right); static Equality_module *merge_func_deps(Equality_module *start, Equality_module *new_fields, Equality_module *end, uint and_level); -static Table_value *get_table_value(Table_elimination *te, TABLE *table); -static Field_value *get_field_value(Table_elimination *te, Field *field); +static Table_value *get_table_value(Func_dep_analyzer *te, TABLE *table); +static Field_value *get_field_value(Func_dep_analyzer *te, Field *field); static void mark_as_eliminated(JOIN *join, TABLE_LIST *tbl); #ifndef DBUG_OFF -static void dbug_print_deps(Table_elimination *te); +static void dbug_print_deps(Func_dep_analyzer *te); #endif /*******************************************************************************************/ @@ -361,17 +363,13 @@ fdeps INOUT Put produced equality conditions here and_level INOUT AND-level (like in add_key_fields) cond Condition to process - usable_tables Tables which fields we're interested in. That is, - Equality_module represent "tbl.col=expr" and we'll - produce them only if tbl is in usable_tables. DESCRIPTION This function is modeled after add_key_fields() */ static -bool build_eq_deps_for_cond(Table_elimination *te, Equality_module **fdeps, - uint *and_level, Item *cond, - table_map usable_tables) +bool build_eq_deps_for_cond(Func_dep_analyzer *te, Equality_module **fdeps, + uint *and_level, Item *cond) { if (cond->type() == Item_func::COND_ITEM) { @@ -384,7 +382,7 @@ Item *item; while ((item=li++)) { - if (build_eq_deps_for_cond(te, fdeps, and_level, item, usable_tables)) + if (build_eq_deps_for_cond(te, fdeps, and_level, item)) return TRUE; } for (; org_key_fields != *fdeps ; org_key_fields++) @@ -393,14 +391,14 @@ else { (*and_level)++; - if (build_eq_deps_for_cond(te, fdeps, and_level, li++, usable_tables)) + if (build_eq_deps_for_cond(te, fdeps, and_level, li++)) return TRUE; Item *item; while ((item=li++)) { Equality_module *start_key_fields= *fdeps; (*and_level)++; - if (build_eq_deps_for_cond(te, fdeps, and_level, item, usable_tables)) + if (build_eq_deps_for_cond(te, fdeps, and_level, item)) return TRUE; *fdeps= merge_func_deps(org_key_fields, start_key_fields, *fdeps, ++(*and_level)); @@ -420,10 +418,8 @@ { if (cond_func->argument_count() == 2) { - if (add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1], - usable_tables) || - add_eq_dep(te, fdeps, *and_level, cond_func, args[1], args[0], - usable_tables)) + if (add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1]) || + add_eq_dep(te, fdeps, *and_level, cond_func, args[1], args[0])) return TRUE; } } @@ -434,10 +430,8 @@ (fld= args[0]->real_item())->type() == Item::FIELD_ITEM && args[1]->eq(args[2], ((Item_field*)fld)->field->binary())) { - if (add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1], - usable_tables) || - add_eq_dep(te, fdeps, *and_level, cond_func, args[1], args[0], - usable_tables)) + if (add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1]) || + add_eq_dep(te, fdeps, *and_level, cond_func, args[1], args[0])) return TRUE; } break; @@ -445,17 +439,14 @@ case Item_func::EQ_FUNC: case Item_func::EQUAL_FUNC: { - add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1], - usable_tables); - add_eq_dep(te, fdeps, *and_level, cond_func, args[1], args[0], - usable_tables); + add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1]); + add_eq_dep(te, fdeps, *and_level, cond_func, args[1], args[0]); break; } case Item_func::ISNULL_FUNC: { Item *tmp=new Item_null; - if (!tmp || add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1], - usable_tables)) + if (!tmp || add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1])) return TRUE; break; } @@ -474,8 +465,7 @@ */ while ((item= it++)) { - if (add_eq_dep(te, fdeps, *and_level, cond_func, item, const_item, - usable_tables)) + if (add_eq_dep(te, fdeps, *and_level, cond_func, item, const_item)) return TRUE; } } @@ -496,8 +486,7 @@ { if (!field->eq(item2->field)) { - if (add_eq_dep(te, fdeps, *and_level, cond_func, item, item2, - usable_tables)) + if (add_eq_dep(te, fdeps, *and_level, cond_func, item, item2)) return TRUE; } } @@ -656,11 +645,10 @@ */ static -bool add_eq_dep(Table_elimination *te, Equality_module **eq_mod, - uint and_level, Item_func *cond, Item *left, Item *right, - table_map usable_tables) +bool add_eq_dep(Func_dep_analyzer *te, Equality_module **eq_mod, + uint and_level, Item_func *cond, Item *left, Item *right) { - if ((left->used_tables() & usable_tables) && + if ((left->used_tables() & te->usable_tables) && !(right->used_tables() & RAND_TABLE_BIT) && left->real_item()->type() == Item::FIELD_ITEM) { @@ -698,7 +686,7 @@ Get a Table_value object for the given table, creating it if necessary. */ -static Table_value *get_table_value(Table_elimination *te, TABLE *table) +static Table_value *get_table_value(Func_dep_analyzer *te, TABLE *table) { Table_value *tbl_dep; if (!(tbl_dep= new Table_value(table))) @@ -724,7 +712,7 @@ Get a Field_value object for the given field, creating it if necessary */ -static Field_value *get_field_value(Table_elimination *te, Field *field) +static Field_value *get_field_value(Func_dep_analyzer *te, Field *field) { TABLE *table= field->table; Table_value *tbl_dep; @@ -762,7 +750,7 @@ class Field_dependency_recorder : public Field_enumerator { public: - Field_dependency_recorder(Table_elimination *te_arg): te(te_arg) + Field_dependency_recorder(Func_dep_analyzer *te_arg): te(te_arg) {} void see_field(Field *field) @@ -792,7 +780,7 @@ } } - Table_elimination *te; + Func_dep_analyzer *te; /* Offset of the expression we're processing in the dependency bitmap */ uint expr_offset; }; @@ -819,7 +807,7 @@ */ static -bool setup_equality_modules_deps(Table_elimination *te, +bool setup_equality_modules_deps(Func_dep_analyzer *te, Module_dep **bound_deps_list) { DBUG_ENTER("setup_equality_modules_deps"); @@ -1089,8 +1077,8 @@ Module_dep *bound_modules; //psergey-todo: move allocs to somewhere else. - Table_elimination pte(join); - Table_elimination *te= &pte; + Func_dep_analyzer pte(join); + Func_dep_analyzer *te= &pte; uint m= max(join->thd->lex->current_select->max_equal_elems,1); uint max_elems= ((join->thd->lex->current_select->cond_count+1)*2 + join->thd->lex->current_select->between_count)*m + 1 + 10; @@ -1118,11 +1106,12 @@ } } + te->usable_tables= dep_tables; /* Analyze the the ON expression and create Equality_module objects and Field_value objects for their left parts. */ - if (build_eq_deps_for_cond(te, &eq_dep, &and_level, cond, dep_tables) || + if (build_eq_deps_for_cond(te, &eq_dep, &and_level, cond) || eq_dep == te->equality_deps) return FALSE; @@ -1158,7 +1147,7 @@ - all its fields are known */ -void Table_value::now_bound(Table_elimination *te, +void Table_value::now_bound(Func_dep_analyzer *te, Module_dep **bound_modules) { DBUG_PRINT("info", ("table %s is now bound", table->alias)); @@ -1184,7 +1173,7 @@ } -void Field_value::now_bound(Table_elimination *te, +void Field_value::now_bound(Func_dep_analyzer *te, Module_dep **bound_modules) { DBUG_PRINT("info", ("field %s.%s is now bound", field->table->alias, @@ -1212,7 +1201,7 @@ Walk through expressions that depend on this field and 'notify' them that this field is no longer unknown. */ -void Field_value::signal_from_field_to_exprs(Table_elimination* te, +void Field_value::signal_from_field_to_exprs(Func_dep_analyzer* te, Module_dep **bound_modules) { for (uint i=0; i < te->n_equality_deps; i++) @@ -1230,7 +1219,7 @@ } -bool Outer_join_module::now_bound(Table_elimination *te, +bool Outer_join_module::now_bound(Func_dep_analyzer *te, Value_dep **bound_values) { DBUG_PRINT("info", ("Outer join eliminated")); @@ -1238,7 +1227,7 @@ } -bool Equality_module::now_bound(Table_elimination *te, +bool Equality_module::now_bound(Func_dep_analyzer *te, Value_dep **bound_values) { /* For field=expr and we got to know the expr, so we know the field */ @@ -1253,7 +1242,7 @@ } /* Unique key is known means its table is known */ -bool Key_module::now_bound(Table_elimination *te, Value_dep **bound_values) +bool Key_module::now_bound(Func_dep_analyzer *te, Value_dep **bound_values) { if (!table->bound) { @@ -1305,7 +1294,7 @@ #ifndef DBUG_OFF static -void dbug_print_deps(Table_elimination *te) +void dbug_print_deps(Func_dep_analyzer *te) { DBUG_ENTER("dbug_print_deps"); DBUG_LOCK_FILE;

1 0

[Maria-developers] Rev 2736: More code cleanups in file:///home/psergey/dev/maria-5.1-table-elim-r11/
by Sergey Petrunya 19 Aug '09

19 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r11/ ------------------------------------------------------------ revno: 2736 revision-id: psergey(a)askmonty.org-20090818221948-f2mg0szfqrve06f9 parent: psergey(a)askmonty.org-20090818211810-48v6pb8dt0sqkysi committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Wed 2009-08-19 01:19:48 +0300 message: More code cleanups === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-18 21:18:10 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-18 22:19:48 +0000 @@ -295,14 +295,16 @@ bzero(table_deps, sizeof(table_deps)); } - JOIN *join; + JOIN *join; /* Join we're working on */ + /* Array of equality dependencies */ Equality_module *equality_deps; uint n_equality_deps; /* Number of elements in the array */ /* tablenr -> Table_value* mapping. */ Table_value *table_deps[MAX_KEY]; - + + /* Element for the outer join we're attempting to eliminate */ Outer_join_module *outer_join_dep; /* Bitmap of how expressions depend on bits */ @@ -312,13 +314,13 @@ void eliminate_tables(JOIN *join); static bool -eliminate_tables_for_list(Table_elimination *te, +eliminate_tables_for_list(JOIN *join, List<TABLE_LIST> *join_list, table_map tables_in_list, Item *on_expr, table_map tables_used_elsewhere); static -bool check_func_dependency(Table_elimination *te, +bool check_func_dependency(JOIN *join, table_map dep_tables, List_iterator<TABLE_LIST> *it, TABLE_LIST *oj_tbl, @@ -753,61 +755,6 @@ /* - Create an Outer_join_module object for the given outer join - - DESCRIPTION - Outer_join_module objects for children (or further descendants) are always - created before the parents. -*/ - -#if 0 -static -Outer_join_module *get_outer_join_dep(Table_elimination *te, - // TABLE_LIST *outer_join, - table_map deps_map) -{ - Outer_join_module *oj_dep; - if (!(oj_dep= new Outer_join_module(/*outer_join, */my_count_bits(deps_map)))) - return NULL; - - /* - Collect a bitmap fo tables that we depend on, and also set parent pointer - for descendant outer join elements. - */ - Table_map_iterator it(deps_map); - int idx; - while ((idx= it.next_bit()) != Table_map_iterator::BITMAP_END) - { - Table_value *table_dep; - if (!(table_dep= te->table_deps[idx])) - { - /* - We get here only when ON expression had no references to inner tables - and Table_map objects weren't created for them. This is a rare/ - unimportant case so it's ok to do not too efficient searches. - */ - TABLE *table= NULL; - for (TABLE_LIST *tlist= te->join->select_lex->leaf_tables; tlist; - tlist=tlist->next_leaf) - { - if (tlist->table->tablenr == (uint)idx) - { - table=tlist->table; - break; - } - } - DBUG_ASSERT(table); - if (!(table_dep= get_table_value(te, table))) - return NULL; - } - table_dep->outer_join_dep= oj_dep; - } - return oj_dep; -} -#endif - - -/* This is used to analyze expressions in "tbl.col=expr" dependencies so that we can figure out which fields the expression depends on. */ @@ -852,33 +799,39 @@ /* - Setup equality dependencies + Setup inbound dependency relationships for tbl.col=expr equalities SYNOPSIS - setup_equality_deps() + setup_equality_modules_deps() te Table elimination context bound_deps_list OUT Start of linked list of elements that were found to be bound (caller will use this to see if that allows to declare further elements bound) DESCRIPTION + Setup inbound dependency relationships for tbl.col=expr equalities: + - allocate a bitmap where we store such dependencies + - for each "tbl.col=expr" equality, analyze the expr part and find out + which fields it refers to and set appropriate dependencies. + RETURN - + FALSE OK + TRUE Out of memory */ static -bool setup_equality_deps(Table_elimination *te, Module_dep **bound_deps_list) +bool setup_equality_modules_deps(Table_elimination *te, + Module_dep **bound_deps_list) { - DBUG_ENTER("setup_equality_deps"); + DBUG_ENTER("setup_equality_modules_deps"); - if (!te->n_equality_deps) - DBUG_RETURN(TRUE); /* - Count Field_value objects and assign each of them a unique bitmap_offset. + Count Field_value objects and assign each of them a unique bitmap_offset + value. */ uint offset= 0; for (Table_value **tbl_dep=te->table_deps; tbl_dep < te->table_deps + MAX_TABLES; - tbl_dep++) // psergey-todo: Wipe this out altogether + tbl_dep++) { if (*tbl_dep) { @@ -924,7 +877,6 @@ } *bound_deps_list= bound_dep; - DBUG_EXECUTE("test", dbug_print_deps(te); ); DBUG_RETURN(FALSE); } @@ -1016,15 +968,7 @@ if (all_tables & ~used_tables) { /* There are some tables that we probably could eliminate. Try it. */ - //psergey-todo: move allocs to somewhere else. - Table_elimination te(join); - uint m= max(thd->lex->current_select->max_equal_elems,1); - uint max_elems= ((thd->lex->current_select->cond_count+1)*2 + - thd->lex->current_select->between_count)*m + 1 + 10; - if (!(te.equality_deps= new Equality_module[max_elems])) - DBUG_VOID_RETURN; - - eliminate_tables_for_list(&te, join->join_list, all_tables, NULL, + eliminate_tables_for_list(join, join->join_list, all_tables, NULL, used_tables); } DBUG_VOID_RETURN; @@ -1056,7 +1000,7 @@ */ static bool -eliminate_tables_for_list(Table_elimination *te, List<TABLE_LIST> *join_list, +eliminate_tables_for_list(JOIN *join, List<TABLE_LIST> *join_list, table_map list_tables, Item *on_expr, table_map tables_used_elsewhere) { @@ -1074,13 +1018,13 @@ if (tbl->nested_join) { /* This is "... LEFT JOIN (join_nest) ON cond" */ - if (eliminate_tables_for_list(te, + if (eliminate_tables_for_list(join, &tbl->nested_join->join_list, tbl->nested_join->used_tables, tbl->on_expr, outside_used_tables)) { - mark_as_eliminated(te->join, tbl); + mark_as_eliminated(join, tbl); } else all_eliminated= FALSE; @@ -1089,10 +1033,10 @@ { /* This is "... LEFT JOIN tbl ON cond" */ if (!(tbl->table->map & outside_used_tables) && - check_func_dependency(te, tbl->table->map, NULL, tbl, + check_func_dependency(join, tbl->table->map, NULL, tbl, tbl->on_expr)) { - mark_as_eliminated(te->join, tbl); + mark_as_eliminated(join, tbl); } else all_eliminated= FALSE; @@ -1109,7 +1053,7 @@ if (all_eliminated && on_expr && !(list_tables & tables_used_elsewhere)) { it.rewind(); - return check_func_dependency(te, list_tables & ~te->join->eliminated_tables, + return check_func_dependency(join, list_tables & ~join->eliminated_tables, &it, NULL, on_expr); } return FALSE; /* not eliminated */ @@ -1135,18 +1079,27 @@ */ static -bool check_func_dependency(Table_elimination *te, +bool check_func_dependency(JOIN *join, table_map dep_tables, List_iterator<TABLE_LIST> *it, TABLE_LIST *oj_tbl, Item* cond) { uint and_level=0; + Module_dep *bound_modules; + + //psergey-todo: move allocs to somewhere else. + Table_elimination pte(join); + Table_elimination *te= &pte; + uint m= max(join->thd->lex->current_select->max_equal_elems,1); + uint max_elems= ((join->thd->lex->current_select->cond_count+1)*2 + + join->thd->lex->current_select->between_count)*m + 1 + 10; + if (!(te->equality_deps= new Equality_module[max_elems])) + return FALSE; + Equality_module* eq_dep= te->equality_deps; - Module_dep *bound_modules; - - bzero(te->table_deps, sizeof(te->table_deps)); - + + /* Create Table_value objects for all tables we're trying to eliminate */ if (oj_tbl) { if (!get_table_value(te, oj_tbl->table)) @@ -1165,19 +1118,25 @@ } } - /* Extract equalities from the ON expression */ + /* + Analyze the the ON expression and create Equality_module objects and + Field_value objects for their left parts. + */ if (build_eq_deps_for_cond(te, &eq_dep, &and_level, cond, dep_tables) || eq_dep == te->equality_deps) return FALSE; - + te->n_equality_deps= eq_dep - te->equality_deps; - /* Create objects for running wave algorithm */ + if (!(te->outer_join_dep= new Outer_join_module(my_count_bits(dep_tables))) || - setup_equality_deps(te, &bound_modules)) + setup_equality_modules_deps(te, &bound_modules)) { return FALSE; /* OOM, default to non-dependent */ } + + DBUG_EXECUTE("test", dbug_print_deps(te); ); + /* The running wave algorithm itself: */ Value_dep *bound_values= NULL; while (bound_modules) {

1 0

[Maria-developers] Rev 2735: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r11/
by Sergey Petrunya 18 Aug '09

18 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r11/ ------------------------------------------------------------ revno: 2735 revision-id: psergey(a)askmonty.org-20090818211810-48v6pb8dt0sqkysi parent: psergey(a)askmonty.org-20090818150151-uef38y50m8m1mgnu committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Wed 2009-08-19 00:18:10 +0300 message: MWL#17: Table elimination - Better comments - Switch from "type" enum and switch to virtual functions for member funcs. === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-18 15:01:51 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-18 21:18:10 +0000 @@ -58,7 +58,7 @@ Table elimination is redone on every PS re-execution. - TABLE ELIMINATION ALGORITHM + TABLE ELIMINATION ALGORITHM FOR ONE OUTER JOIN As said above, we can remove inner side of an outer join if it is @@ -101,14 +101,14 @@ each value is either bound (i.e. functionally dependent) or not. Module nodes: - - Nodes representing tblX.colY=expr equalities. Equality node has + - Modules representing tblX.colY=expr equalities. Equality module has = incoming edges from columns used in expr = outgoing edge to tblX.colY column. - Nodes representing unique keys. Unique key has - = incoming edges from key component value nodes - = outgoing edge to key's table node - - Inner side of outer join node. Outer join node has - = incoming edges from table value nodes + = incoming edges from key component value modules + = outgoing edge to key's table module + - Inner side of outer join module. Outer join module has + = incoming edges from table value modules = No outgoing edges. Once we reach it, we know we can eliminate the outer join. A module may depend on multiple values, and hence its primary attribute is @@ -116,10 +116,13 @@ The algorithm starts with equality nodes that don't have any incoming edges (their expressions are either constant or depend only on tables that are - outside of any outer joins) and proceeds to traverse dependency->dependant - edges until we've other traversed everything (TODO rephrase elaborate), or - we've reached the point where all outer join modules have zero unsatisfied - dependencies. + outside of the outer join in question) and performns a breadth-first + traversal. If we reach the outer join nest node, it means outer join is + functionally-dependant and can be eliminated. Otherwise it cannot. + + HANDLING MULTIPLE NESTED OUTER JOINS + (TODO : explanations why 'local bottom up is sufficient') + */ class Value_dep; @@ -143,13 +146,9 @@ class Value_dep : public Sql_alloc { public: - enum { - VALUE_FIELD, - VALUE_TABLE, - } type; /* Type of the object */ - - Value_dep(): bound(FALSE), next(NULL) - {} + Value_dep(): bound(FALSE), next(NULL) {} + virtual void now_bound(Table_elimination *te, Module_dep **bound_modules)=0; + virtual ~Value_dep() {} /* only to shut up compiler warnings */ bool bound; Value_dep *next; @@ -166,22 +165,28 @@ public: Field_value(Table_value *table_arg, Field *field_arg) : table(table_arg), field(field_arg) - { - type= Value_dep::VALUE_FIELD; - } + {} Table_value *table; /* Table this field is from */ Field *field; /* - Field_deps that belong to one table form a linked list. list members are - ordered by field_index + Field_deps that belong to one table form a linked list, ordered by + field_index */ Field_value *next_table_field; uint bitmap_offset; /* Offset of our part of the bitmap */ + + /* + Field became known. Check out + - unique keys we belong to + - expressions that depend on us. + */ + void now_bound(Table_elimination *te, Module_dep **bound_modules); + void signal_from_field_to_exprs(Table_elimination* te, + Module_dep **bound_modules); }; - /* A table value. There is one Table_value object for every table that can potentially be eliminated. @@ -192,14 +197,13 @@ { public: Table_value(TABLE *table_arg) : - table(table_arg), fields(NULL), keys(NULL), outer_join_dep(NULL) - { - type= Value_dep::VALUE_TABLE; - } + table(table_arg), fields(NULL), keys(NULL) + {} TABLE *table; Field_value *fields; /* Ordered list of fields that belong to this table */ Key_module *keys; /* Ordered list of Unique keys in this table */ - Outer_join_module *outer_join_dep; /* Innermost eliminable outer join we're in */ + //Outer_join_module *outer_join_dep; + void now_bound(Table_elimination *te, Module_dep **bound_modules); }; @@ -211,12 +215,11 @@ { public: enum { - MODULE_EXPRESSION, - MODULE_MULTI_EQUALITY, - MODULE_UNIQUE_KEY, MODULE_OUTER_JOIN } type; /* Type of the object */ + virtual bool now_bound(Table_elimination *te, Value_dep **bound_modules)=0; + virtual ~Module_dep(){} /* Used to make a linked list of elements that became bound and thus can make elements that depend on them bound, too. @@ -240,6 +243,7 @@ /* Used during condition analysis only, similar to KEYUSE::level */ uint level; + bool now_bound(Table_elimination *te, Value_dep **bound_values); }; @@ -254,17 +258,16 @@ Key_module(Table_value *table_arg, uint keyno_arg, uint n_parts_arg) : table(table_arg), keyno(keyno_arg), next_table_key(NULL) { - type= Module_dep::MODULE_UNIQUE_KEY; unknown_args= n_parts_arg; } Table_value *table; /* Table this key is from */ uint keyno; /* Unique keys form a linked list, ordered by keyno */ Key_module *next_table_key; + bool now_bound(Table_elimination *te, Value_dep **bound_values); }; - /* An outer join nest that is subject to elimination - it depends on all tables inside it @@ -273,18 +276,11 @@ class Outer_join_module: public Module_dep { public: - Outer_join_module(//TABLE_LIST *table_list_arg, - uint n_children) - // table_list(table_list_arg) + Outer_join_module(uint n_children) { - type= Module_dep::MODULE_OUTER_JOIN; unknown_args= n_children; } - /* - Outer join we're representing. This can be a join nest or one table that - is outer join'ed. - */ -// TABLE_LIST *table_list; + bool now_bound(Table_elimination *te, Value_dep **bound_values); }; @@ -307,6 +303,8 @@ /* tablenr -> Table_value* mapping. */ Table_value *table_deps[MAX_KEY]; + Outer_join_module *outer_join_dep; + /* Bitmap of how expressions depend on bits */ MY_BITMAP expr_deps; }; @@ -319,8 +317,12 @@ table_map tables_in_list, Item *on_expr, table_map tables_used_elsewhere); -static bool -check_func_dependency(Table_elimination *te, table_map tables, Item* cond); +static +bool check_func_dependency(Table_elimination *te, + table_map dep_tables, + List_iterator<TABLE_LIST> *it, + TABLE_LIST *oj_tbl, + Item* cond); static bool build_eq_deps_for_cond(Table_elimination *te, Equality_module **fdeps, @@ -332,16 +334,12 @@ Item_func *cond, Item *left, Item *right, table_map usable_tables); static -Equality_module *merge_func_deps(Equality_module *start, Equality_module *new_fields, - Equality_module *end, uint and_level); +Equality_module *merge_func_deps(Equality_module *start, + Equality_module *new_fields, + Equality_module *end, uint and_level); static Table_value *get_table_value(Table_elimination *te, TABLE *table); static Field_value *get_field_value(Table_elimination *te, Field *field); -static Outer_join_module *get_outer_join_dep(Table_elimination *te, - //TABLE_LIST *outer_join, - table_map deps_map); -static -bool run_elimination_wave(Table_elimination *te, Module_dep *bound_modules); static void mark_as_eliminated(JOIN *join, TABLE_LIST *tbl); @@ -560,7 +558,7 @@ static Equality_module *merge_func_deps(Equality_module *start, Equality_module *new_fields, - Equality_module *end, uint and_level) + Equality_module *end, uint and_level) { if (start == new_fields) return start; // Impossible or @@ -684,7 +682,6 @@ } } - (*eq_mod)->type= Module_dep::MODULE_EXPRESSION; //psergey-todo; if (!((*eq_mod)->field= get_field_value(te, field))) return TRUE; (*eq_mod)->expression= right; @@ -763,6 +760,7 @@ created before the parents. */ +#if 0 static Outer_join_module *get_outer_join_dep(Table_elimination *te, // TABLE_LIST *outer_join, @@ -806,6 +804,7 @@ } return oj_dep; } +#endif /* @@ -879,7 +878,7 @@ uint offset= 0; for (Table_value **tbl_dep=te->table_deps; tbl_dep < te->table_deps + MAX_TABLES; - tbl_dep++) // psergey-todo: TODO change to Table_map_iterator + tbl_dep++) // psergey-todo: Wipe this out altogether { if (*tbl_dep) { @@ -1090,7 +1089,8 @@ { /* This is "... LEFT JOIN tbl ON cond" */ if (!(tbl->table->map & outside_used_tables) && - check_func_dependency(te, tbl->table->map, tbl->on_expr)) + check_func_dependency(te, tbl->table->map, NULL, tbl, + tbl->on_expr)) { mark_as_eliminated(te->join, tbl); } @@ -1108,8 +1108,9 @@ /* Try eliminating the nest we're called for */ if (all_eliminated && on_expr && !(list_tables & tables_used_elsewhere)) { + it.rewind(); return check_func_dependency(te, list_tables & ~te->join->eliminated_tables, - on_expr); + &it, NULL, on_expr); } return FALSE; /* not eliminated */ } @@ -1134,31 +1135,130 @@ */ static -bool check_func_dependency(Table_elimination *te, table_map tables, Item* cond) +bool check_func_dependency(Table_elimination *te, + table_map dep_tables, + List_iterator<TABLE_LIST> *it, + TABLE_LIST *oj_tbl, + Item* cond) { uint and_level=0; Equality_module* eq_dep= te->equality_deps; - if (build_eq_deps_for_cond(te, &eq_dep, &and_level, cond, tables)) - return TRUE; + Module_dep *bound_modules; + + bzero(te->table_deps, sizeof(te->table_deps)); + + if (oj_tbl) + { + if (!get_table_value(te, oj_tbl->table)) + return FALSE; + } + else + { + TABLE_LIST *tbl; + while ((tbl= (*it)++)) + { + if (tbl->table && (tbl->table->map & dep_tables)) + { + if (!get_table_value(te, tbl->table)) + return FALSE; + } + } + } + + /* Extract equalities from the ON expression */ + if (build_eq_deps_for_cond(te, &eq_dep, &and_level, cond, dep_tables) || + eq_dep == te->equality_deps) + return FALSE; + te->n_equality_deps= eq_dep - te->equality_deps; - Module_dep *bound_modules; - if (!get_outer_join_dep(te, tables) && - !setup_equality_deps(te, &bound_modules) && - run_elimination_wave(te, bound_modules)) - { - return TRUE; /* eliminated */ - } - return FALSE; -} - - -static -void signal_from_field_to_exprs(Table_elimination* te, Field_value *field_dep, - Module_dep **bound_modules) + /* Create objects for running wave algorithm */ + if (!(te->outer_join_dep= new Outer_join_module(my_count_bits(dep_tables))) || + setup_equality_deps(te, &bound_modules)) + { + return FALSE; /* OOM, default to non-dependent */ + } + + Value_dep *bound_values= NULL; + while (bound_modules) + { + for (;bound_modules; bound_modules= bound_modules->next) + { + if (bound_modules->now_bound(te, &bound_values)) + return TRUE; /* Dependent! */ + } + for (;bound_values; bound_values=bound_values->next) + bound_values->now_bound(te, &bound_modules); + } + return FALSE; /* Not dependent */ +} + + +/* + Table is known means that + - one more element in outer join nest is known + - all its fields are known +*/ + +void Table_value::now_bound(Table_elimination *te, + Module_dep **bound_modules) +{ + DBUG_PRINT("info", ("table %s is now bound", table->alias)); + + for (Field_value *field_dep= fields; field_dep; + field_dep= field_dep->next_table_field) + { + if (!field_dep->bound) + { + /* Mark as bound and add to the list */ + field_dep->bound= TRUE; + field_dep->signal_from_field_to_exprs(te, bound_modules); + } + } + + if (te->outer_join_dep->unknown_args && + !--te->outer_join_dep->unknown_args) + { + /* Mark as bound and add to the list */ + te->outer_join_dep->next= *bound_modules; + *bound_modules= te->outer_join_dep; + } +} + + +void Field_value::now_bound(Table_elimination *te, + Module_dep **bound_modules) +{ + DBUG_PRINT("info", ("field %s.%s is now bound", field->table->alias, + field->field_name)); + + for (Key_module *key_dep= table->keys; key_dep; + key_dep= key_dep->next_table_key) + { + if (field->part_of_key.is_set(key_dep->keyno) && + key_dep->unknown_args && !--key_dep->unknown_args) + { + DBUG_PRINT("info", ("key %s.%s is now bound", + key_dep->table->table->alias, + key_dep->table->table->key_info[key_dep->keyno].name)); + /* Mark as bound and add to the list */ + key_dep->next= *bound_modules; + *bound_modules= key_dep; + } + } + signal_from_field_to_exprs(te, bound_modules); +} + + +/* + Walk through expressions that depend on this field and 'notify' them + that this field is no longer unknown. +*/ +void Field_value::signal_from_field_to_exprs(Table_elimination* te, + Module_dep **bound_modules) { for (uint i=0; i < te->n_equality_deps; i++) { - if (bitmap_is_set(&te->expr_deps, field_dep->bitmap_offset + i) && + if (bitmap_is_set(&te->expr_deps, bitmap_offset + i) && te->equality_deps[i].unknown_args && !--te->equality_deps[i].unknown_args) { @@ -1171,131 +1271,37 @@ } -/* - Run the wave. - All Func_dep-derived objects are divided into three classes: - - Those that have bound=FALSE - - Those that have bound=TRUE - - Those that have bound=TRUE and are in the list.. -*/ - -static -bool run_elimination_wave(Table_elimination *te, Module_dep *bound_modules) -{ - Value_dep *bound_values= NULL; - while (bound_modules) - { - for (;bound_modules; bound_modules= bound_modules->next) - { - switch (bound_modules->type) - { - case Module_dep::MODULE_EXPRESSION: - { - /* It's a field=expr and we got to know the expr, so we know the field */ - Equality_module *eq_dep= (Equality_module*)bound_modules; - if (!eq_dep->field->bound) - { - /* Mark as bound and add to the list */ - eq_dep->field->bound= TRUE; - eq_dep->field->next= bound_values; - bound_values= eq_dep->field; - } - break; - } - case Module_dep::MODULE_UNIQUE_KEY: - { - /* Unique key is known means the table is known */ - Table_value *table_dep=((Key_module*)bound_modules)->table; - if (!table_dep->bound) - { - /* Mark as bound and add to the list */ - table_dep->bound= TRUE; - table_dep->next= bound_values; - bound_values= table_dep; - } - break; - } - case Module_dep::MODULE_OUTER_JOIN: - { - DBUG_PRINT("info", ("Outer join eliminated")); - return TRUE; - break; - } - case Module_dep::MODULE_MULTI_EQUALITY: - default: - DBUG_ASSERT(0); - } - } - - for (;bound_values; bound_values=bound_values->next) - { - switch (bound_values->type) - { - case Value_dep::VALUE_FIELD: - { - /* - Field became known. Check out - - unique keys we belong to - - expressions that depend on us. - */ - Field_value *field_dep= (Field_value*)bound_values; - DBUG_PRINT("info", ("field %s.%s is now bound", - field_dep->field->table->alias, - field_dep->field->field_name)); - - for (Key_module *key_dep= field_dep->table->keys; key_dep; - key_dep= key_dep->next_table_key) - { - if (field_dep->field->part_of_key.is_set(key_dep->keyno) && - key_dep->unknown_args && !--key_dep->unknown_args) - { - DBUG_PRINT("info", ("key %s.%s is now bound", - key_dep->table->table->alias, - key_dep->table->table->key_info[key_dep->keyno].name)); - /* Mark as bound and add to the list */ - key_dep->next= bound_modules; - bound_modules= key_dep; - } - } - signal_from_field_to_exprs(te, field_dep, &bound_modules); - break; - } - case Value_dep::VALUE_TABLE: - { - Table_value *table_dep=(Table_value*)bound_values; - DBUG_PRINT("info", ("table %s is now bound", - table_dep->table->alias)); - /* - Table is known means that - - one more element in outer join nest is known - - all its fields are known - */ - Outer_join_module *outer_join_dep= table_dep->outer_join_dep; - if (outer_join_dep->unknown_args && - !--outer_join_dep->unknown_args) - { - /* Mark as bound and add to the list */ - outer_join_dep->next= bound_modules; - bound_modules= outer_join_dep; - break; - } - - for (Field_value *field_dep= table_dep->fields; field_dep; - field_dep= field_dep->next_table_field) - { - if (!field_dep->bound) - { - /* Mark as bound and add to the list */ - field_dep->bound= TRUE; - signal_from_field_to_exprs(te, field_dep, &bound_modules); - } - } - break; - } - default: - DBUG_ASSERT(0); - } - } +bool Outer_join_module::now_bound(Table_elimination *te, + Value_dep **bound_values) +{ + DBUG_PRINT("info", ("Outer join eliminated")); + return TRUE; /* Signal to finish the process */ +} + + +bool Equality_module::now_bound(Table_elimination *te, + Value_dep **bound_values) +{ + /* For field=expr and we got to know the expr, so we know the field */ + if (!field->bound) + { + /* Mark as bound and add to the list */ + field->bound= TRUE; + field->next= *bound_values; + *bound_values= field; + } + return FALSE; +} + +/* Unique key is known means its table is known */ +bool Key_module::now_bound(Table_elimination *te, Value_dep **bound_values) +{ + if (!table->bound) + { + /* Mark as bound and add to the list */ + table->bound= TRUE; + table->next= *bound_values; + *bound_values= table; } return FALSE; } @@ -1304,6 +1310,7 @@ /* Mark one table or the whole join nest as eliminated. */ + static void mark_as_eliminated(JOIN *join, TABLE_LIST *tbl) { TABLE *table;

1 0

[Maria-developers] Rev 2734: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r11/
by Sergey Petrunya 18 Aug '09

18 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r11/ ------------------------------------------------------------ revno: 2734 revision-id: psergey(a)askmonty.org-20090818150151-uef38y50m8m1mgnu parent: psergey(a)askmonty.org-20090818130358-akd84j4m2i91lw5a committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Tue 2009-08-18 18:01:51 +0300 message: MWL#17: Table elimination - Code cleanup === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-18 13:03:58 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-18 15:01:51 +0000 @@ -275,7 +275,7 @@ public: Outer_join_module(//TABLE_LIST *table_list_arg, uint n_children) - // table_list(table_list_arg), parent(NULL) + // table_list(table_list_arg) { type= Module_dep::MODULE_OUTER_JOIN; unknown_args= n_children; @@ -285,9 +285,6 @@ is outer join'ed. */ // TABLE_LIST *table_list; - - /* Parent eliminable outer join, if any */ -// Outer_join_module *parent; }; @@ -297,7 +294,7 @@ class Table_elimination { public: - Table_elimination(JOIN *join_arg) : join(join_arg), n_outer_joins(0) + Table_elimination(JOIN *join_arg) : join(join_arg) { bzero(table_deps, sizeof(table_deps)); } @@ -310,14 +307,20 @@ /* tablenr -> Table_value* mapping. */ Table_value *table_deps[MAX_KEY]; - /* Outer joins that are candidates for elimination */ - List<Outer_join_module> oj_deps; - uint n_outer_joins; - /* Bitmap of how expressions depend on bits */ MY_BITMAP expr_deps; }; +void eliminate_tables(JOIN *join); + +static bool +eliminate_tables_for_list(Table_elimination *te, + List<TABLE_LIST> *join_list, + table_map tables_in_list, + Item *on_expr, + table_map tables_used_elsewhere); +static bool +check_func_dependency(Table_elimination *te, table_map tables, Item* cond); static bool build_eq_deps_for_cond(Table_elimination *te, Equality_module **fdeps, @@ -339,9 +342,10 @@ table_map deps_map); static bool run_elimination_wave(Table_elimination *te, Module_dep *bound_modules); -void eliminate_tables(JOIN *join); static void mark_as_eliminated(JOIN *join, TABLE_LIST *tbl); + + #ifndef DBUG_OFF static void dbug_print_deps(Table_elimination *te); #endif @@ -383,10 +387,6 @@ if (build_eq_deps_for_cond(te, fdeps, and_level, item, usable_tables)) return TRUE; } - /* - TODO: inject here a "if we have {t.col=const AND t.col=smth_else}, then - remove the second part" logic. - */ for (; org_key_fields != *fdeps ; org_key_fields++) org_key_fields->level= *and_level; } @@ -614,7 +614,7 @@ /* Ok, the results are within the [start, first_free) range, and the useful - elements have level==and_level. Now, lets remove all unusable elements: + elements have level==and_level. Now, remove all unusable elements: */ for (Equality_module *old=start ; old != first_free ;) { @@ -632,14 +632,31 @@ /* - Add an Equality_module element for a given predicate, if applicable + Add an Equality_module element for left=right condition + + SYNOPSIS + add_eq_dep() + te Table elimination context + eq_mod INOUT Store created Equality_module here and increment ptr if + you do so + and_level AND-level () + cond Condition we've inferred the left=right equality from. + left Left expression + right Right expression + usable_tables Create Equality_module only if Left_expression's table + belongs to this set. DESCRIPTION - This function is modeled after add_key_field(). + Check if the passed equality means that 'left' expr is functionally dependent on + the 'right', and if yes, create an Equality_module object for it. + + RETURN + FALSE OK + TRUE Out of memory */ static -bool add_eq_dep(Table_elimination *te, Equality_module **eq_dep, +bool add_eq_dep(Table_elimination *te, Equality_module **eq_mod, uint and_level, Item_func *cond, Item *left, Item *right, table_map usable_tables) { @@ -658,8 +675,8 @@ else { /* - We can't use indexes if the effective collation - of the operation differ from the field collation. + We can't assume there's a functional dependency if the effective + collation of the operation differ from the field collation. */ if (field->cmp_type() == STRING_RESULT && ((Field_str*)field)->charset() != cond->compare_collation()) @@ -667,12 +684,12 @@ } } - (*eq_dep)->type= Module_dep::MODULE_EXPRESSION; //psergey-todo; - if (!((*eq_dep)->field= get_field_value(te, field))) + (*eq_mod)->type= Module_dep::MODULE_EXPRESSION; //psergey-todo; + if (!((*eq_mod)->field= get_field_value(te, field))) return TRUE; - (*eq_dep)->expression= right; - (*eq_dep)->level= and_level; - (*eq_dep)++; + (*eq_mod)->expression= right; + (*eq_mod)->level= and_level; + (*eq_mod)++; } return FALSE; } @@ -754,7 +771,6 @@ Outer_join_module *oj_dep; if (!(oj_dep= new Outer_join_module(/*outer_join, */my_count_bits(deps_map)))) return NULL; - te->n_outer_joins++; /* Collect a bitmap fo tables that we depend on, and also set parent pointer @@ -786,25 +802,7 @@ if (!(table_dep= get_table_value(te, table))) return NULL; } - - /* - Walk from the table up to its embedding outer joins. The goal is to - find the least embedded outer join nest and set its parent pointer to - point to the newly created Outer_join_module. - to set the pointer of its near - */ - //if (!table_dep->outer_join_dep) - table_dep->outer_join_dep= oj_dep; - /* - else - { - Outer_join_module *oj= table_dep->outer_join_dep; - while (oj->parent) - oj= oj->parent; - if (oj != oj_dep) - oj->parent=oj_dep; - } - */ + table_dep->outer_join_dep= oj_dep; } return oj_dep; } @@ -815,10 +813,10 @@ that we can figure out which fields the expression depends on. */ -class Field_dependency_setter : public Field_enumerator +class Field_dependency_recorder : public Field_enumerator { public: - Field_dependency_setter(Table_elimination *te_arg): te(te_arg) + Field_dependency_recorder(Table_elimination *te_arg): te(te_arg) {} void see_field(Field *field) @@ -856,13 +854,16 @@ /* Setup equality dependencies - + SYNOPSIS setup_equality_deps() te Table elimination context bound_deps_list OUT Start of linked list of elements that were found to be bound (caller will use this to see if that allows to declare further elements bound) + DESCRIPTION + RETURN + */ static @@ -907,15 +908,15 @@ Also collect a linked list of equalities that are bound. */ Module_dep *bound_dep= NULL; - Field_dependency_setter deps_setter(te); + Field_dependency_recorder deps_recorder(te); for (Equality_module *eq_dep= te->equality_deps; eq_dep < te->equality_deps + te->n_equality_deps; eq_dep++) { - deps_setter.expr_offset= eq_dep - te->equality_deps; + deps_recorder.expr_offset= eq_dep - te->equality_deps; eq_dep->unknown_args= 0; eq_dep->expression->walk(&Item::check_column_usage_processor, FALSE, - (uchar*)&deps_setter); + (uchar*)&deps_recorder); if (!eq_dep->unknown_args) { eq_dep->next= bound_dep; @@ -935,12 +936,11 @@ SYNOPSIS eliminate_tables() join Join to work on - const_tbl_count INOUT Number of constant tables (this includes - eliminated tables) - const_tables INOUT Bitmap of constant tables DESCRIPTION - This function is the entry point for table elimination. + This is the entry point for table elimination. Grep for MODULE INTERFACE + section in this file for calling convention. + The idea behind table elimination is that if we have an outer join: SELECT * FROM t1 LEFT JOIN @@ -968,12 +968,6 @@ See the OVERVIEW section at the top of this file. */ -static uint -eliminate_tables_for_list(Table_elimination *te, - List<TABLE_LIST> *join_list, - Item *on_expr, - table_map tables_in_list, - table_map tables_used_elsewhere); void eliminate_tables(JOIN *join) { @@ -1023,101 +1017,54 @@ if (all_tables & ~used_tables) { /* There are some tables that we probably could eliminate. Try it. */ + //psergey-todo: move allocs to somewhere else. Table_elimination te(join); uint m= max(thd->lex->current_select->max_equal_elems,1); uint max_elems= ((thd->lex->current_select->cond_count+1)*2 + thd->lex->current_select->between_count)*m + 1 + 10; if (!(te.equality_deps= new Equality_module[max_elems])) DBUG_VOID_RETURN; - //Equality_module *eq_deps_end= te.equality_deps; - //table_map eliminable_tables= 0; - /* - if (collect_funcdeps_for_join_list(&te, join->join_list, - FALSE, - used_tables, - &eliminable_tables, - &eq_deps)) - DBUG_VOID_RETURN; - */ - eliminate_tables_for_list(&te, join->join_list, - NULL, - (table_map(1) << join->tables) - 1, + + eliminate_tables_for_list(&te, join->join_list, all_tables, NULL, used_tables); - - /*te.n_equality_deps= eq_deps_end - te.equality_deps; - - Module_dep *bound_modules; - //Value_dep *bound_values; - if (setup_equality_deps(&te, &bound_modules)) - DBUG_VOID_RETURN; - - run_elimination_wave(&te, bound_modules); - */ } DBUG_VOID_RETURN; } -//////////////////////////// - /* Perform table elimination in a given join list SYNOPSIS eliminate_tables_for_list() - join The join - leaves_arr OUT Store here an array of leaf (base) tables that - are descendants of the join_list, and increment - the pointer to point right above the array. - join_list Join list to work on - its_outer_join TRUE <=> join_list is an inner side of an outer - join - FALSE <=> otherwise (this is top-level join list) - tables_in_list Bitmap of tables embedded in the join_list. + te Table elimination context + join_list Join list to work on + list_tables Bitmap of tables embedded in the join_list. + on_expr ON expression, if the join list is the inner side + of an outer join. + NULL means it's not an outer join but rather a + top-level join list. tables_used_elsewhere Bitmap of tables that are referred to from somewhere outside of the join list (e.g. - select list, HAVING, etc). + select list, HAVING, other ON expressions, etc). DESCRIPTION - Perform table elimination for a join list. - Try eliminating children nests first. - The "all tables in join nest can produce only one matching record - combination" property checking is modeled after constant table detection, - plus we reuse info attempts to eliminate child join nests. - + Perform table elimination in a given join list. + RETURN - Number of children left after elimination. 0 means everything was - eliminated. + TRUE The entire join list eliminated + FALSE Join list wasn't eliminated (but some of its possibly were) */ -bool try_eliminating(Table_elimination *te, table_map tables_in_list, - Item* on_expr) -{ - uint and_level=0; - Equality_module* eq_dep= te->equality_deps; - build_eq_deps_for_cond(te, &eq_dep, &and_level, on_expr, - tables_in_list); - te->n_equality_deps= eq_dep - te->equality_deps; - Module_dep *bound_modules; - get_outer_join_dep(te, tables_in_list); - - if (!setup_equality_deps(te, &bound_modules) && - run_elimination_wave(te, bound_modules)) - return TRUE; // eliminated - return FALSE; -} - - -static uint +static bool eliminate_tables_for_list(Table_elimination *te, List<TABLE_LIST> *join_list, - Item *on_expr, - table_map tables_in_list, + table_map list_tables, Item *on_expr, table_map tables_used_elsewhere) { TABLE_LIST *tbl; List_iterator<TABLE_LIST> it(*join_list); table_map tables_used_on_left= 0; - bool not_eliminated= FALSE; + bool all_eliminated= TRUE; while ((tbl= it++)) { @@ -1130,26 +1077,25 @@ /* This is "... LEFT JOIN (join_nest) ON cond" */ if (eliminate_tables_for_list(te, &tbl->nested_join->join_list, + tbl->nested_join->used_tables, tbl->on_expr, - tbl->nested_join->used_tables, outside_used_tables)) { mark_as_eliminated(te->join, tbl); } else - not_eliminated= TRUE; - + all_eliminated= FALSE; } else { /* This is "... LEFT JOIN tbl ON cond" */ if (!(tbl->table->map & outside_used_tables) && - try_eliminating(te, tbl->table->map, tbl->on_expr)) + check_func_dependency(te, tbl->table->map, tbl->on_expr)) { mark_as_eliminated(te->join, tbl); } else - not_eliminated= TRUE; + all_eliminated= FALSE; } tables_used_on_left |= tbl->on_expr->used_tables(); } @@ -1160,98 +1106,51 @@ } /* Try eliminating the nest we're called for */ - if (!not_eliminated && on_expr && !(tables_in_list & tables_used_elsewhere)) - return try_eliminating(te, tables_in_list & ~te->join->eliminated_tables, - on_expr); - + if (all_eliminated && on_expr && !(list_tables & tables_used_elsewhere)) + { + return check_func_dependency(te, list_tables & ~te->join->eliminated_tables, + on_expr); + } return FALSE; /* not eliminated */ } -#if 0 + /* - Build functional dependency graph for elements of given join list + Check if condition makes the a set of tables functionally-dependent SYNOPSIS - collect_funcdeps_for_join_list() - te Table elimination context. - join_list Join list to work on - build_eq_deps TRUE <=> build Equality_module elements for all - members of the join list, even if they cannot - be individually eliminated - tables_used_elsewhere Bitmap of tables that are referred to from - somewhere outside of this join list (e.g. - select list, HAVING, ON expressions of parent - joins, etc). - eliminable_tables INOUT Tables that can potentially be eliminated - (needed so we know for which tables to build - dependencies for) - eq_dep INOUT End of array of equality dependencies. + check_func_dependency() + te Table elimination context + tables Set of tables we want to be functionally dependent + cond Condition to use DESCRIPTION - . + Check if condition allows to conclude that the table set is functionally + dependent on everything else. + + RETURN + TRUE - Yes, functionally dependent + FALSE - No, or error */ -static bool -collect_funcdeps_for_join_list(Table_elimination *te, - List<TABLE_LIST> *join_list, - bool build_eq_deps, - table_map tables_used_elsewhere, - table_map *eliminable_tables, - Equality_module **eq_dep) +static +bool check_func_dependency(Table_elimination *te, table_map tables, Item* cond) { - TABLE_LIST *tbl; - List_iterator<TABLE_LIST> it(*join_list); - table_map tables_used_on_left= 0; - - while ((tbl= it++)) + uint and_level=0; + Equality_module* eq_dep= te->equality_deps; + if (build_eq_deps_for_cond(te, &eq_dep, &and_level, cond, tables)) + return TRUE; + te->n_equality_deps= eq_dep - te->equality_deps; + Module_dep *bound_modules; + if (!get_outer_join_dep(te, tables) && + !setup_equality_deps(te, &bound_modules) && + run_elimination_wave(te, bound_modules)) { - if (tbl->on_expr) - { - table_map outside_used_tables= tables_used_elsewhere | - tables_used_on_left; - bool eliminable; - table_map cur_map; - if (tbl->nested_join) - { - /* This is "... LEFT JOIN (join_nest) ON cond" */ - cur_map= tbl->nested_join->used_tables; - eliminable= !(cur_map & outside_used_tables); - if (eliminable) - *eliminable_tables |= cur_map; - if (collect_funcdeps_for_join_list(te, &tbl->nested_join->join_list, - eliminable || build_eq_deps, - outside_used_tables, - eliminable_tables, - eq_dep)) - return TRUE; - } - else - { - /* This is "... LEFT JOIN tbl ON cond" */ - cur_map= tbl->table->map; - eliminable= !(tbl->table->map & outside_used_tables); - *eliminable_tables |= cur_map; - } - - if (eliminable || build_eq_deps) - { - // build comp_cond from ON expression - uint and_level=0; - build_eq_deps_for_cond(te, eq_dep, &and_level, tbl->on_expr, - *eliminable_tables); - } - - if (eliminable && !get_outer_join_dep(te, tbl, cur_map)) - return TRUE; - - tables_used_on_left |= tbl->on_expr->used_tables(); - } + return TRUE; /* eliminated */ } return FALSE; } -#endif -//////////////////////////// static void signal_from_field_to_exprs(Table_elimination* te, Field_value *field_dep, @@ -1272,18 +1171,18 @@ } +/* + Run the wave. + All Func_dep-derived objects are divided into three classes: + - Those that have bound=FALSE + - Those that have bound=TRUE + - Those that have bound=TRUE and are in the list.. +*/ + static bool run_elimination_wave(Table_elimination *te, Module_dep *bound_modules) { Value_dep *bound_values= NULL; - /* - Run the wave. - All Func_dep-derived objects are divided into three classes: - - Those that have bound=FALSE - - Those that have bound=TRUE - - Those that have bound=TRUE and are in the list.. - - */ while (bound_modules) { for (;bound_modules; bound_modules= bound_modules->next) @@ -1318,14 +1217,8 @@ } case Module_dep::MODULE_OUTER_JOIN: { - //Outer_join_module *outer_join_dep= (Outer_join_module*)bound_modules; - //mark_as_eliminated(te->join, outer_join_dep->table_list); - //if (!--te->n_outer_joins) - { - DBUG_PRINT("info", ("Table elimination eliminated everything" - " it theoretically could")); - return TRUE; - } + DBUG_PRINT("info", ("Outer join eliminated")); + return TRUE; break; } case Module_dep::MODULE_MULTI_EQUALITY: @@ -1373,10 +1266,20 @@ DBUG_PRINT("info", ("table %s is now bound", table_dep->table->alias)); /* - Table is known means + Table is known means that + - one more element in outer join nest is known - all its fields are known - - one more element in outer join nest is known */ + Outer_join_module *outer_join_dep= table_dep->outer_join_dep; + if (outer_join_dep->unknown_args && + !--outer_join_dep->unknown_args) + { + /* Mark as bound and add to the list */ + outer_join_dep->next= bound_modules; + bound_modules= outer_join_dep; + break; + } + for (Field_value *field_dep= table_dep->fields; field_dep; field_dep= field_dep->next_table_field) { @@ -1387,18 +1290,6 @@ signal_from_field_to_exprs(te, field_dep, &bound_modules); } } - //for ( - Outer_join_module *outer_join_dep= table_dep->outer_join_dep; - // outer_join_dep; outer_join_dep= outer_join_dep->parent) - { - if (outer_join_dep->unknown_args && - !--outer_join_dep->unknown_args) - { - /* Mark as bound and add to the list */ - outer_join_dep->next= bound_modules; - bound_modules= outer_join_dep; - } - } break; } default:

1 0

[Maria-developers] Updated (by Psergey): Add EXPLAIN for UPDATE/DELETE (51)
by worklog-noreply＠askmonty.org 18 Aug '09

18 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add EXPLAIN for UPDATE/DELETE CREATION DATE..: Tue, 18 Aug 2009, 16:24 SUPERVISOR.....: Bothorsen IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Client-BackLog TASK ID........: 51 (http://askmonty.org/worklog/?tid=51) VERSION........: Benchmarks-3.0 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Tue, 18 Aug 2009, 16:25)=-=- High-Level Specification modified. --- /tmp/wklog.51.old.12524 2009-08-18 16:25:52.000000000 +0300 +++ /tmp/wklog.51.new.12524 2009-08-18 16:25:52.000000000 +0300 @@ -1 +1,22 @@ +User interface +-------------- +EXPLAIN UPDATE ... and EXPLAIN DELETE ... statements will work and produce +a tabular output similar to EXPLAIN SELECT. + +Implementation +-------------- +The primary challenge will be to change UPDATE/DELETE code to first produce +action plan and then act on it (and not make decisions on the go as it +currently does). + +What EXPLAIN will show +---------------------- +* multi-table one will show the SELECT part as usual +* single-table statements will show an equivalent of access method. + +Besides that, we want to know +- if sorting will be used +- whether the UPDATE will occur on the fly or not. + +(print or not "Using filesort"...) DESCRIPTION: Add support for EXPLAIN UPDATE/DELETE (we'd like support for all potentially long statements, e.g. for ALTER TABLE, but this WL entry is limited to UPDATE/DELETE as they are most often requested, and code that handles them is similar and distinct from other statements). HIGH-LEVEL SPECIFICATION: User interface -------------- EXPLAIN UPDATE ... and EXPLAIN DELETE ... statements will work and produce a tabular output similar to EXPLAIN SELECT. Implementation -------------- The primary challenge will be to change UPDATE/DELETE code to first produce action plan and then act on it (and not make decisions on the go as it currently does). What EXPLAIN will show ---------------------- * multi-table one will show the SELECT part as usual * single-table statements will show an equivalent of access method. Besides that, we want to know - if sorting will be used - whether the UPDATE will occur on the fly or not. (print or not "Using filesort"...) ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Add EXPLAIN for UPDATE/DELETE (51)
by worklog-noreply＠askmonty.org 18 Aug '09

18 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add EXPLAIN for UPDATE/DELETE CREATION DATE..: Tue, 18 Aug 2009, 16:24 SUPERVISOR.....: Bothorsen IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Client-BackLog TASK ID........: 51 (http://askmonty.org/worklog/?tid=51) VERSION........: Benchmarks-3.0 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Tue, 18 Aug 2009, 16:25)=-=- High-Level Specification modified. --- /tmp/wklog.51.old.12524 2009-08-18 16:25:52.000000000 +0300 +++ /tmp/wklog.51.new.12524 2009-08-18 16:25:52.000000000 +0300 @@ -1 +1,22 @@ +User interface +-------------- +EXPLAIN UPDATE ... and EXPLAIN DELETE ... statements will work and produce +a tabular output similar to EXPLAIN SELECT. + +Implementation +-------------- +The primary challenge will be to change UPDATE/DELETE code to first produce +action plan and then act on it (and not make decisions on the go as it +currently does). + +What EXPLAIN will show +---------------------- +* multi-table one will show the SELECT part as usual +* single-table statements will show an equivalent of access method. + +Besides that, we want to know +- if sorting will be used +- whether the UPDATE will occur on the fly or not. + +(print or not "Using filesort"...) DESCRIPTION: Add support for EXPLAIN UPDATE/DELETE (we'd like support for all potentially long statements, e.g. for ALTER TABLE, but this WL entry is limited to UPDATE/DELETE as they are most often requested, and code that handles them is similar and distinct from other statements). HIGH-LEVEL SPECIFICATION: User interface -------------- EXPLAIN UPDATE ... and EXPLAIN DELETE ... statements will work and produce a tabular output similar to EXPLAIN SELECT. Implementation -------------- The primary challenge will be to change UPDATE/DELETE code to first produce action plan and then act on it (and not make decisions on the go as it currently does). What EXPLAIN will show ---------------------- * multi-table one will show the SELECT part as usual * single-table statements will show an equivalent of access method. Besides that, we want to know - if sorting will be used - whether the UPDATE will occur on the fly or not. (print or not "Using filesort"...) ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Psergey): Add EXPLAIN for UPDATE/DELETE (51)
by worklog-noreply＠askmonty.org 18 Aug '09

18 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add EXPLAIN for UPDATE/DELETE CREATION DATE..: Tue, 18 Aug 2009, 16:24 SUPERVISOR.....: Bothorsen IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Client-BackLog TASK ID........: 51 (http://askmonty.org/worklog/?tid=51) VERSION........: Benchmarks-3.0 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: DESCRIPTION: Add support for EXPLAIN UPDATE/DELETE (we'd like support for all potentially long statements, e.g. for ALTER TABLE, but this WL entry is limited to UPDATE/DELETE as they are most often requested, and code that handles them is similar and distinct from other statements). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Psergey): Add EXPLAIN for UPDATE/DELETE (51)
by worklog-noreply＠askmonty.org 18 Aug '09

18 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add EXPLAIN for UPDATE/DELETE CREATION DATE..: Tue, 18 Aug 2009, 16:24 SUPERVISOR.....: Bothorsen IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Client-BackLog TASK ID........: 51 (http://askmonty.org/worklog/?tid=51) VERSION........: Benchmarks-3.0 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: DESCRIPTION: Add support for EXPLAIN UPDATE/DELETE (we'd like support for all potentially long statements, e.g. for ALTER TABLE, but this WL entry is limited to UPDATE/DELETE as they are most often requested, and code that handles them is similar and distinct from other statements). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Rev 2733: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r11/
by Sergey Petrunya 18 Aug '09

18 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r11/ ------------------------------------------------------------ revno: 2733 revision-id: psergey(a)askmonty.org-20090818130358-akd84j4m2i91lw5a parent: psergey(a)askmonty.org-20090817160724-fmmrmwp8zorzn82q committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r11 timestamp: Tue 2009-08-18 16:03:58 +0300 message: MWL#17: Table elimination - Switch from trying to eliminate all tables at once (which didn't work) to the original approach of bottom-up elimination. === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-17 15:02:29 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-18 13:03:58 +0000 @@ -273,8 +273,9 @@ class Outer_join_module: public Module_dep { public: - Outer_join_module(TABLE_LIST *table_list_arg, uint n_children) : - table_list(table_list_arg), parent(NULL) + Outer_join_module(//TABLE_LIST *table_list_arg, + uint n_children) + // table_list(table_list_arg), parent(NULL) { type= Module_dep::MODULE_OUTER_JOIN; unknown_args= n_children; @@ -283,10 +284,10 @@ Outer join we're representing. This can be a join nest or one table that is outer join'ed. */ - TABLE_LIST *table_list; +// TABLE_LIST *table_list; /* Parent eliminable outer join, if any */ - Outer_join_module *parent; +// Outer_join_module *parent; }; @@ -334,10 +335,10 @@ static Table_value *get_table_value(Table_elimination *te, TABLE *table); static Field_value *get_field_value(Table_elimination *te, Field *field); static Outer_join_module *get_outer_join_dep(Table_elimination *te, - TABLE_LIST *outer_join, + //TABLE_LIST *outer_join, table_map deps_map); static -void run_elimination_wave(Table_elimination *te, Module_dep *bound_modules); +bool run_elimination_wave(Table_elimination *te, Module_dep *bound_modules); void eliminate_tables(JOIN *join); static void mark_as_eliminated(JOIN *join, TABLE_LIST *tbl); @@ -357,7 +358,7 @@ and_level INOUT AND-level (like in add_key_fields) cond Condition to process usable_tables Tables which fields we're interested in. That is, - Equality_dep represent "tbl.col=expr" and we'll + Equality_module represent "tbl.col=expr" and we'll produce them only if tbl is in usable_tables. DESCRIPTION This function is modeled after add_key_fields() @@ -747,11 +748,11 @@ static Outer_join_module *get_outer_join_dep(Table_elimination *te, - TABLE_LIST *outer_join, + // TABLE_LIST *outer_join, table_map deps_map) { Outer_join_module *oj_dep; - if (!(oj_dep= new Outer_join_module(outer_join, my_count_bits(deps_map)))) + if (!(oj_dep= new Outer_join_module(/*outer_join, */my_count_bits(deps_map)))) return NULL; te->n_outer_joins++; @@ -792,8 +793,9 @@ point to the newly created Outer_join_module. to set the pointer of its near */ - if (!table_dep->outer_join_dep) + //if (!table_dep->outer_join_dep) table_dep->outer_join_dep= oj_dep; + /* else { Outer_join_module *oj= table_dep->outer_join_dep; @@ -802,95 +804,13 @@ if (oj != oj_dep) oj->parent=oj_dep; } + */ } return oj_dep; } /* - Build functional dependency graph for elements of given join list - - SYNOPSIS - collect_funcdeps_for_join_list() - te Table elimination context. - join_list Join list to work on - build_eq_deps TRUE <=> build Equality_module elements for all - members of the join list, even if they cannot - be individually eliminated - tables_used_elsewhere Bitmap of tables that are referred to from - somewhere outside of this join list (e.g. - select list, HAVING, ON expressions of parent - joins, etc). - eliminable_tables INOUT Tables that can potentially be eliminated - (needed so we know for which tables to build - dependencies for) - eq_dep INOUT End of array of equality dependencies. - - DESCRIPTION - . -*/ - -static bool -collect_funcdeps_for_join_list(Table_elimination *te, - List<TABLE_LIST> *join_list, - bool build_eq_deps, - table_map tables_used_elsewhere, - table_map *eliminable_tables, - Equality_module **eq_dep) -{ - TABLE_LIST *tbl; - List_iterator<TABLE_LIST> it(*join_list); - table_map tables_used_on_left= 0; - - while ((tbl= it++)) - { - if (tbl->on_expr) - { - table_map outside_used_tables= tables_used_elsewhere | - tables_used_on_left; - bool eliminable; - table_map cur_map; - if (tbl->nested_join) - { - /* This is "... LEFT JOIN (join_nest) ON cond" */ - cur_map= tbl->nested_join->used_tables; - eliminable= !(cur_map & outside_used_tables); - if (eliminable) - *eliminable_tables |= cur_map; - if (collect_funcdeps_for_join_list(te, &tbl->nested_join->join_list, - eliminable || build_eq_deps, - outside_used_tables, - eliminable_tables, - eq_dep)) - return TRUE; - } - else - { - /* This is "... LEFT JOIN tbl ON cond" */ - cur_map= tbl->table->map; - eliminable= !(tbl->table->map & outside_used_tables); - *eliminable_tables |= cur_map; - } - - if (eliminable || build_eq_deps) - { - // build comp_cond from ON expression - uint and_level=0; - build_eq_deps_for_cond(te, eq_dep, &and_level, tbl->on_expr, - *eliminable_tables); - } - - if (eliminable && !get_outer_join_dep(te, tbl, cur_map)) - return TRUE; - - tables_used_on_left |= tbl->on_expr->used_tables(); - } - } - return FALSE; -} - - -/* This is used to analyze expressions in "tbl.col=expr" dependencies so that we can figure out which fields the expression depends on. */ @@ -950,13 +870,15 @@ { DBUG_ENTER("setup_equality_deps"); + if (!te->n_equality_deps) + DBUG_RETURN(TRUE); /* Count Field_value objects and assign each of them a unique bitmap_offset. */ uint offset= 0; for (Table_value **tbl_dep=te->table_deps; tbl_dep < te->table_deps + MAX_TABLES; - tbl_dep++) + tbl_dep++) // psergey-todo: TODO change to Table_map_iterator { if (*tbl_dep) { @@ -1046,6 +968,12 @@ See the OVERVIEW section at the top of this file. */ +static uint +eliminate_tables_for_list(Table_elimination *te, + List<TABLE_LIST> *join_list, + Item *on_expr, + table_map tables_in_list, + table_map tables_used_elsewhere); void eliminate_tables(JOIN *join) { @@ -1101,15 +1029,22 @@ thd->lex->current_select->between_count)*m + 1 + 10; if (!(te.equality_deps= new Equality_module[max_elems])) DBUG_VOID_RETURN; - Equality_module *eq_deps_end= te.equality_deps; - table_map eliminable_tables= 0; + //Equality_module *eq_deps_end= te.equality_deps; + //table_map eliminable_tables= 0; + /* if (collect_funcdeps_for_join_list(&te, join->join_list, FALSE, used_tables, &eliminable_tables, - &eq_deps_end)) + &eq_deps)) DBUG_VOID_RETURN; - te.n_equality_deps= eq_deps_end - te.equality_deps; + */ + eliminate_tables_for_list(&te, join->join_list, + NULL, + (table_map(1) << join->tables) - 1, + used_tables); + + /*te.n_equality_deps= eq_deps_end - te.equality_deps; Module_dep *bound_modules; //Value_dep *bound_values; @@ -1117,11 +1052,207 @@ DBUG_VOID_RETURN; run_elimination_wave(&te, bound_modules); + */ } DBUG_VOID_RETURN; } +//////////////////////////// + +/* + Perform table elimination in a given join list + + SYNOPSIS + eliminate_tables_for_list() + join The join + leaves_arr OUT Store here an array of leaf (base) tables that + are descendants of the join_list, and increment + the pointer to point right above the array. + join_list Join list to work on + its_outer_join TRUE <=> join_list is an inner side of an outer + join + FALSE <=> otherwise (this is top-level join list) + tables_in_list Bitmap of tables embedded in the join_list. + tables_used_elsewhere Bitmap of tables that are referred to from + somewhere outside of the join list (e.g. + select list, HAVING, etc). + + DESCRIPTION + Perform table elimination for a join list. + Try eliminating children nests first. + The "all tables in join nest can produce only one matching record + combination" property checking is modeled after constant table detection, + plus we reuse info attempts to eliminate child join nests. + + RETURN + Number of children left after elimination. 0 means everything was + eliminated. +*/ + +bool try_eliminating(Table_elimination *te, table_map tables_in_list, + Item* on_expr) +{ + uint and_level=0; + Equality_module* eq_dep= te->equality_deps; + build_eq_deps_for_cond(te, &eq_dep, &and_level, on_expr, + tables_in_list); + te->n_equality_deps= eq_dep - te->equality_deps; + Module_dep *bound_modules; + get_outer_join_dep(te, tables_in_list); + + if (!setup_equality_deps(te, &bound_modules) && + run_elimination_wave(te, bound_modules)) + return TRUE; // eliminated + return FALSE; +} + + +static uint +eliminate_tables_for_list(Table_elimination *te, List<TABLE_LIST> *join_list, + Item *on_expr, + table_map tables_in_list, + table_map tables_used_elsewhere) +{ + TABLE_LIST *tbl; + List_iterator<TABLE_LIST> it(*join_list); + table_map tables_used_on_left= 0; + bool not_eliminated= FALSE; + + while ((tbl= it++)) + { + if (tbl->on_expr) + { + table_map outside_used_tables= tables_used_elsewhere | + tables_used_on_left; + if (tbl->nested_join) + { + /* This is "... LEFT JOIN (join_nest) ON cond" */ + if (eliminate_tables_for_list(te, + &tbl->nested_join->join_list, + tbl->on_expr, + tbl->nested_join->used_tables, + outside_used_tables)) + { + mark_as_eliminated(te->join, tbl); + } + else + not_eliminated= TRUE; + + } + else + { + /* This is "... LEFT JOIN tbl ON cond" */ + if (!(tbl->table->map & outside_used_tables) && + try_eliminating(te, tbl->table->map, tbl->on_expr)) + { + mark_as_eliminated(te->join, tbl); + } + else + not_eliminated= TRUE; + } + tables_used_on_left |= tbl->on_expr->used_tables(); + } + else + { + DBUG_ASSERT(!tbl->nested_join); + } + } + + /* Try eliminating the nest we're called for */ + if (!not_eliminated && on_expr && !(tables_in_list & tables_used_elsewhere)) + return try_eliminating(te, tables_in_list & ~te->join->eliminated_tables, + on_expr); + + return FALSE; /* not eliminated */ +} + +#if 0 +/* + Build functional dependency graph for elements of given join list + + SYNOPSIS + collect_funcdeps_for_join_list() + te Table elimination context. + join_list Join list to work on + build_eq_deps TRUE <=> build Equality_module elements for all + members of the join list, even if they cannot + be individually eliminated + tables_used_elsewhere Bitmap of tables that are referred to from + somewhere outside of this join list (e.g. + select list, HAVING, ON expressions of parent + joins, etc). + eliminable_tables INOUT Tables that can potentially be eliminated + (needed so we know for which tables to build + dependencies for) + eq_dep INOUT End of array of equality dependencies. + + DESCRIPTION + . +*/ + +static bool +collect_funcdeps_for_join_list(Table_elimination *te, + List<TABLE_LIST> *join_list, + bool build_eq_deps, + table_map tables_used_elsewhere, + table_map *eliminable_tables, + Equality_module **eq_dep) +{ + TABLE_LIST *tbl; + List_iterator<TABLE_LIST> it(*join_list); + table_map tables_used_on_left= 0; + + while ((tbl= it++)) + { + if (tbl->on_expr) + { + table_map outside_used_tables= tables_used_elsewhere | + tables_used_on_left; + bool eliminable; + table_map cur_map; + if (tbl->nested_join) + { + /* This is "... LEFT JOIN (join_nest) ON cond" */ + cur_map= tbl->nested_join->used_tables; + eliminable= !(cur_map & outside_used_tables); + if (eliminable) + *eliminable_tables |= cur_map; + if (collect_funcdeps_for_join_list(te, &tbl->nested_join->join_list, + eliminable || build_eq_deps, + outside_used_tables, + eliminable_tables, + eq_dep)) + return TRUE; + } + else + { + /* This is "... LEFT JOIN tbl ON cond" */ + cur_map= tbl->table->map; + eliminable= !(tbl->table->map & outside_used_tables); + *eliminable_tables |= cur_map; + } + + if (eliminable || build_eq_deps) + { + // build comp_cond from ON expression + uint and_level=0; + build_eq_deps_for_cond(te, eq_dep, &and_level, tbl->on_expr, + *eliminable_tables); + } + + if (eliminable && !get_outer_join_dep(te, tbl, cur_map)) + return TRUE; + + tables_used_on_left |= tbl->on_expr->used_tables(); + } + } + return FALSE; +} +#endif + +//////////////////////////// + static void signal_from_field_to_exprs(Table_elimination* te, Field_value *field_dep, Module_dep **bound_modules) @@ -1142,7 +1273,7 @@ static -void run_elimination_wave(Table_elimination *te, Module_dep *bound_modules) +bool run_elimination_wave(Table_elimination *te, Module_dep *bound_modules) { Value_dep *bound_values= NULL; /* @@ -1187,13 +1318,13 @@ } case Module_dep::MODULE_OUTER_JOIN: { - Outer_join_module *outer_join_dep= (Outer_join_module*)bound_modules; - mark_as_eliminated(te->join, outer_join_dep->table_list); - if (!--te->n_outer_joins) + //Outer_join_module *outer_join_dep= (Outer_join_module*)bound_modules; + //mark_as_eliminated(te->join, outer_join_dep->table_list); + //if (!--te->n_outer_joins) { DBUG_PRINT("info", ("Table elimination eliminated everything" " it theoretically could")); - return; + return TRUE; } break; } @@ -1256,8 +1387,9 @@ signal_from_field_to_exprs(te, field_dep, &bound_modules); } } - for (Outer_join_module *outer_join_dep= table_dep->outer_join_dep; - outer_join_dep; outer_join_dep= outer_join_dep->parent) + //for ( + Outer_join_module *outer_join_dep= table_dep->outer_join_dep; + // outer_join_dep; outer_join_dep= outer_join_dep->parent) { if (outer_join_dep->unknown_args && !--outer_join_dep->unknown_args) @@ -1274,6 +1406,7 @@ } } } + return FALSE; }

1 0

[Maria-developers] Rev 2732: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r10/
by Sergey Petrunya 17 Aug '09

17 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r10/ ------------------------------------------------------------ revno: 2732 revision-id: psergey(a)askmonty.org-20090817160724-fmmrmwp8zorzn82q parent: psergey(a)askmonty.org-20090817150229-jy461nqbmk8nzhha committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Mon 2009-08-17 19:07:24 +0300 message: MWL#17: Table elimination - More testcases === modified file 'mysql-test/r/table_elim.result' --- a/mysql-test/r/table_elim.result 2009-08-17 15:02:29 +0000 +++ b/mysql-test/r/table_elim.result 2009-08-17 16:07:24 +0000 @@ -218,4 +218,48 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL NULL NULL NULL NULL 2 1 SIMPLE t2 eq_ref PRIMARY PRIMARY 4 test.t1.col 1 +drop table t1, t2, t3; +# +# Check things that look like functional dependencies but really are not +# +create table t1 (a char(10) character set latin1 collate latin1_general_ci primary key); +insert into t1 values ('foo'); +insert into t1 values ('bar'); +create table t2 (a char(10) character set latin1 collate latin1_general_cs primary key); +insert into t2 values ('foo'); +insert into t2 values ('FOO'); +this must not use table elimination: +explain select t1.* from t1 left join t2 on t2.a='foo' collate latin1_general_ci; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index NULL PRIMARY 10 NULL 2 Using index +1 SIMPLE t2 index PRIMARY PRIMARY 10 NULL 2 Using index +this must not use table elimination: +explain select t1.* from t1 left join t2 on t2.a=t1.a collate latin1_general_ci; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index NULL PRIMARY 10 NULL 2 Using index +1 SIMPLE t2 index PRIMARY PRIMARY 10 NULL 2 Using index +drop table t1,t2; +create table t1 (a int primary key); +insert into t1 values (1),(2); +create table t2 (a char(10) primary key); +insert into t2 values ('1'),('1.0'); +this must not use table elimination: +explain select t1.* from t1 left join t2 on t2.a=1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index NULL PRIMARY 4 NULL 2 Using index +1 SIMPLE t2 index PRIMARY PRIMARY 10 NULL 2 Using index +this must not use table elimination: +explain select t1.* from t1 left join t2 on t2.a=t1.a; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index NULL PRIMARY 4 NULL 2 Using index +1 SIMPLE t2 index PRIMARY PRIMARY 10 NULL 2 Using index +drop table t1, t2; +create table t1 (a char(10) primary key); +insert into t1 values ('foo'),('bar'); +create table t2 (a char(10), unique key(a(2))); +insert into t2 values ('foo'),('bar'); +explain select t1.* from t1 left join t2 on t2.a=t1.a; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index NULL PRIMARY 10 NULL 2 Using index +1 SIMPLE t2 ref a a 3 test.t1.a 2 drop table t1, t2; === modified file 'mysql-test/t/table_elim.test' --- a/mysql-test/t/table_elim.test 2009-08-17 15:02:29 +0000 +++ b/mysql-test/t/table_elim.test 2009-08-17 16:07:24 +0000 @@ -175,5 +175,46 @@ explain select t1.*, t2.* from t1 left join (t2 left join t3 on t3.pk=t2.col) on t2.pk=t1.col; +drop table t1, t2, t3; + +--echo # +--echo # Check things that look like functional dependencies but really are not +--echo # + +create table t1 (a char(10) character set latin1 collate latin1_general_ci primary key); +insert into t1 values ('foo'); +insert into t1 values ('bar'); + +create table t2 (a char(10) character set latin1 collate latin1_general_cs primary key); +insert into t2 values ('foo'); +insert into t2 values ('FOO'); + +-- echo this must not use table elimination: +explain select t1.* from t1 left join t2 on t2.a='foo' collate latin1_general_ci; + +-- echo this must not use table elimination: +explain select t1.* from t1 left join t2 on t2.a=t1.a collate latin1_general_ci; +drop table t1,t2; + +create table t1 (a int primary key); +insert into t1 values (1),(2); +create table t2 (a char(10) primary key); +insert into t2 values ('1'),('1.0'); +-- echo this must not use table elimination: +explain select t1.* from t1 left join t2 on t2.a=1; +-- echo this must not use table elimination: +explain select t1.* from t1 left join t2 on t2.a=t1.a; + +drop table t1, t2; +# partial unique keys do not work at the moment, although they are able to +# provide one-match guarantees: +create table t1 (a char(10) primary key); +insert into t1 values ('foo'),('bar'); + +create table t2 (a char(10), unique key(a(2))); +insert into t2 values ('foo'),('bar'); + +explain select t1.* from t1 left join t2 on t2.a=t1.a; + drop table t1, t2;

1 0

[Maria-developers] Rev 2731: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r10/
by Sergey Petrunya 17 Aug '09

17 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r10/ ------------------------------------------------------------ revno: 2731 revision-id: psergey(a)askmonty.org-20090817150229-jy461nqbmk8nzhha parent: psergey(a)askmonty.org-20090816180159-z3lfkjpjfsm7zbp0 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Mon 2009-08-17 18:02:29 +0300 message: MWL#17: Table elimination - More dbug printouts - More testcases === modified file 'mysql-test/r/table_elim.result' --- a/mysql-test/r/table_elim.result 2009-06-29 13:51:15 +0000 +++ b/mysql-test/r/table_elim.result 2009-08-17 15:02:29 +0000 @@ -202,3 +202,20 @@ id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t1 ALL NULL NULL NULL NULL 4 drop table t1, t2; +create table t1 (pk int primary key, col int); +insert into t1 values (1,1),(2,2); +create table t2 like t1; +insert into t2 select * from t1; +create table t3 like t1; +insert into t3 select * from t1; +explain +select t1.* from t1 left join ( t2 left join t3 on t3.pk=t2.col) on t2.col=t1.col; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 2 +1 SIMPLE t2 ALL NULL NULL NULL NULL 2 +explain +select t1.*, t2.* from t1 left join (t2 left join t3 on t3.pk=t2.col) on t2.pk=t1.col; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 2 +1 SIMPLE t2 eq_ref PRIMARY PRIMARY 4 test.t1.col 1 +drop table t1, t2; === modified file 'mysql-test/t/table_elim.test' --- a/mysql-test/t/table_elim.test 2009-06-29 13:51:15 +0000 +++ b/mysql-test/t/table_elim.test 2009-08-17 15:02:29 +0000 @@ -157,4 +157,23 @@ t2.pk3=t2.pk1; drop table t1, t2; +# +# Check that equality propagation is taken into account +# +create table t1 (pk int primary key, col int); +insert into t1 values (1,1),(2,2); + +create table t2 like t1; +insert into t2 select * from t1; + +create table t3 like t1; +insert into t3 select * from t1; + +explain +select t1.* from t1 left join ( t2 left join t3 on t3.pk=t2.col) on t2.col=t1.col; + +explain +select t1.*, t2.* from t1 left join (t2 left join t3 on t3.pk=t2.col) on t2.pk=t1.col; + +drop table t1, t2; === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-16 18:01:59 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-17 15:02:29 +0000 @@ -136,7 +136,8 @@ /* - A value. + A value, something that can be bound or not bound. Also, values can be linked + in a list. */ class Value_dep : public Sql_alloc @@ -203,7 +204,7 @@ /* - A 'module' + A 'module'. Module has dependencies */ class Module_dep : public Sql_alloc @@ -227,7 +228,6 @@ }; - /* A "tbl.column= expr" equality dependency. tbl.column depends on fields used in expr. @@ -333,6 +333,9 @@ static Table_value *get_table_value(Table_elimination *te, TABLE *table); static Field_value *get_field_value(Table_elimination *te, Field *field); +static Outer_join_module *get_outer_join_dep(Table_elimination *te, + TABLE_LIST *outer_join, + table_map deps_map); static void run_elimination_wave(Table_elimination *te, Module_dep *bound_modules); void eliminate_tables(JOIN *join); @@ -1212,15 +1215,19 @@ - expressions that depend on us. */ Field_value *field_dep= (Field_value*)bound_values; + DBUG_PRINT("info", ("field %s.%s is now bound", + field_dep->field->table->alias, + field_dep->field->field_name)); + for (Key_module *key_dep= field_dep->table->keys; key_dep; key_dep= key_dep->next_table_key) { - DBUG_PRINT("info", ("key %s.%s is now bound", - key_dep->table->table->alias, - key_dep->table->table->key_info[key_dep->keyno].name)); if (field_dep->field->part_of_key.is_set(key_dep->keyno) && key_dep->unknown_args && !--key_dep->unknown_args) { + DBUG_PRINT("info", ("key %s.%s is now bound", + key_dep->table->table->alias, + key_dep->table->table->key_info[key_dep->keyno].name)); /* Mark as bound and add to the list */ key_dep->next= bound_modules; bound_modules= key_dep;

1 0

[Maria-developers] MWL#39 improving mysqlbinlog output and doing rename
by Michael Widenius 17 Aug '09

17 Aug '09

Hi! >>>>> "Kristian" == Kristian Nielsen <knielsen(a)knielsen-hq.org> writes: Kristian> Sergey Petrunya <psergey(a)askmonty.org> writes: >> I've worked through the list and filed >> >> MWL#39 Replication tasks >> >> and its subordinate tasks: >> >> MWL#36 Add a mysqlbinlog option to change the used database >> MWL#37 Add an option to mysqlbinlog to produce SQL script with fewer roundtrips >> MWL#38 Make mysqlbinlog not to output unneeded statements when using --database >> MWL#40 Add a mysqlbinlog option to filter updates to certain tables >> MWL#41 Add a mysqlbinlog option to filter certain kinds of statements Kristian> Thanks, Sergey! I've started to work my way through them and the many Kristian> associated issues to understand for this. Kristian> One idea that occured to me and that I would like to bounce off of you others Kristian> (while I still continue thinking more about it and also working on the Kristian> original proposals): Kristian> In some sense the root of the problem is the magic "BINLOG" statement, which Kristian> really is not very nice. It is much much harder than it should be to see from Kristian> the binlog what data is actually changed from row-based events. Kristian> If the BINLOG statement was using proper syntax showing the actual data Kristian> changes, then the original customer problem would likely be solved as they Kristian> could treat it the same way as statement-based replication events. This is Kristian> especially nice when one considers that even row-based replication uses lots Kristian> of statement-based events, not to mention mixed-mode replication. Kristian> But readable BINLOG statements would be very nice in any case, and solve a Kristian> much more general problem than these worklogs, in an arguably nicer way. Note that it would also be very nice to have in the binlog the exact original statement: - Easier to understand why updates happened. - Would make it easier to find out where in application things went wrong (as you can search for exact strings) - Allow one to filter things based on comments in the statement. The cost would be to have an approximately 2 as big binlog. (Especially insert of big blob's would be a bit painful). Kristian> There are also a number of potential problems that need careful consideration Kristian> of course. I haven't yet fully thought things through, just wanted to throw up Kristian> the idea. Kristian> I went throught the code for RBR events, and came up with the following Kristian> possible syntax: Kristian> BINLOG Kristian> WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 Kristian> TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 Kristian> TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 Kristian> WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 Kristian> UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), Kristian> FROM ('a') TO ('b') FLAGS 0x0 Kristian> DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; Kristian> This is basically a dump of what is stored in the events, and would be an Kristian> alternative to BINLOG 'gwWEShMBAA...'. Kristian> So what do people think? The above would be a much better option than using the current syntax. Kristian> The implementation of this is not necessarily all that much harder than the Kristian> suggested worklogs under MWL#39 I think. The server upon reading this would Kristian> just reconstruct the binary representation of the binlog and proceed as the Kristian> old BINLOG statement. The mysqlbinlog program would just print out the Kristian> fields. The main complications are the addition of syntax to the Bison grammer Kristian> plus the need to handle all the possible types. Kristian> If we choose the easiest option in all the MWL#39 subtasks that would probably Kristian> be somewhat easier. On the other hand this would be a much more generally Kristian> useful feature, and would make trivial a lot of the suggested modifications to Kristian> mysqlbinlog. Which of the original customer problems would the above solve ? For exampling, doing general rename of databases wouldn't be much easier to do with the the above syntax (as we still need to handle SBR). Kristian> I'm pretty sure this would be easier than some of the harder options in the Kristian> MWL#39 subtasks. Which one are you thinking about ? (I like the proposed syntax, but don't grasp why things would be simple when doing this ). Kristian> So I need to think a bit more about this to make up my mind if I like it. But Kristian> I think at least it is a very interesting idea. Kristian> Comments? Kristian> (Also, isn't it time to move the discussion to the public Kristian> maria-developers(a)lists.launchpad.net list?) Done. Regards, Monty

3 6

[Maria-developers] Updated (by Psergey): mysqlbinlog: remove undesired effects of format description binlog statement (50)
by worklog-noreply＠askmonty.org 17 Aug '09

17 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: mysqlbinlog: remove undesired effects of format description binlog statement CREATION DATE..: Mon, 17 Aug 2009, 14:12 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 50 (http://askmonty.org/worklog/?tid=50) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.11389 2009-08-17 14:13:05.000000000 +0300 +++ /tmp/wklog.50.new.11389 2009-08-17 14:13:05.000000000 +0300 @@ -1 +1,14 @@ +First item +---------- +AFAIU what needs to be done is: +1. record a source server version (it is in the first binlog event). +2. don't emit a BINLOG statement if the recorded version number is 5.0.x or + below. + +and we'll get a 5.0-applicable binlog. + +Second item +----------- +One question that one needs to sort out before disabling server_id change is +why it was put there in the first place? Should it be always removed? DESCRIPTION: According to complaints in BUG#46640: 1. mysqlbinlog will emit a 5.0-incompatible "format description binlog statement" even when reading a binary log that was produced by 5.0. This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. 2. When one applies "format description binlog statement" at the slave, it will change the slave's server_id when applied. This WL is to fix these issues. HIGH-LEVEL SPECIFICATION: First item ---------- AFAIU what needs to be done is: 1. record a source server version (it is in the first binlog event). 2. don't emit a BINLOG statement if the recorded version number is 5.0.x or below. and we'll get a 5.0-applicable binlog. Second item ----------- One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): mysqlbinlog: remove undesired effects of format description binlog statement (50)
by worklog-noreply＠askmonty.org 17 Aug '09

17 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: mysqlbinlog: remove undesired effects of format description binlog statement CREATION DATE..: Mon, 17 Aug 2009, 14:12 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 50 (http://askmonty.org/worklog/?tid=50) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Mon, 17 Aug 2009, 14:13)=-=- High-Level Specification modified. --- /tmp/wklog.50.old.11389 2009-08-17 14:13:05.000000000 +0300 +++ /tmp/wklog.50.new.11389 2009-08-17 14:13:05.000000000 +0300 @@ -1 +1,14 @@ +First item +---------- +AFAIU what needs to be done is: +1. record a source server version (it is in the first binlog event). +2. don't emit a BINLOG statement if the recorded version number is 5.0.x or + below. + +and we'll get a 5.0-applicable binlog. + +Second item +----------- +One question that one needs to sort out before disabling server_id change is +why it was put there in the first place? Should it be always removed? DESCRIPTION: According to complaints in BUG#46640: 1. mysqlbinlog will emit a 5.0-incompatible "format description binlog statement" even when reading a binary log that was produced by 5.0. This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. 2. When one applies "format description binlog statement" at the slave, it will change the slave's server_id when applied. This WL is to fix these issues. HIGH-LEVEL SPECIFICATION: First item ---------- AFAIU what needs to be done is: 1. record a source server version (it is in the first binlog event). 2. don't emit a BINLOG statement if the recorded version number is 5.0.x or below. and we'll get a 5.0-applicable binlog. Second item ----------- One question that one needs to sort out before disabling server_id change is why it was put there in the first place? Should it be always removed? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Psergey): mysqlbinlog: remove undesired effects of format description binlog statement (50)
by worklog-noreply＠askmonty.org 17 Aug '09

17 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: mysqlbinlog: remove undesired effects of format description binlog statement CREATION DATE..: Mon, 17 Aug 2009, 14:12 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 50 (http://askmonty.org/worklog/?tid=50) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: DESCRIPTION: According to complaints in BUG#46640: 1. mysqlbinlog will emit a 5.0-incompatible "format description binlog statement" even when reading a binary log that was produced by 5.0. This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. 2. When one applies "format description binlog statement" at the slave, it will change the slave's server_id when applied. This WL is to fix these issues. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Psergey): mysqlbinlog: remove undesired effects of format description binlog statement (50)
by worklog-noreply＠askmonty.org 17 Aug '09

17 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: mysqlbinlog: remove undesired effects of format description binlog statement CREATION DATE..: Mon, 17 Aug 2009, 14:12 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 50 (http://askmonty.org/worklog/?tid=50) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: DESCRIPTION: According to complaints in BUG#46640: 1. mysqlbinlog will emit a 5.0-incompatible "format description binlog statement" even when reading a binary log that was produced by 5.0. This will cause an error when one tires to apply mysqlbinlog output to a 5.0.x server. 2. When one applies "format description binlog statement" at the slave, it will change the slave's server_id when applied. This WL is to fix these issues. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Add a mysqlbinlog option to produce succint output (45)
by worklog-noreply＠askmonty.org 17 Aug '09

17 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add a mysqlbinlog option to produce succint output CREATION DATE..: Sat, 15 Aug 2009, 23:40 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 45 (http://askmonty.org/worklog/?tid=45) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Mon, 17 Aug 2009, 13:58)=-=- High Level Description modified. --- /tmp/wklog.45.old.10768 2009-08-17 13:58:07.000000000 +0300 +++ /tmp/wklog.45.new.10768 2009-08-17 13:58:07.000000000 +0300 @@ -4,3 +4,5 @@ This will be different from --short-form option. That option causes mysqlbinlog not to print RBR events, i.e. the output is not supposed to be applied. +There is also option to have --short-form do what this WL requires, see MySQL +BUG#18337. -=-=(Guest - Mon, 17 Aug 2009, 11:17)=-=- Dependency created: 39 now depends on 45 -=-=(Guest - Mon, 17 Aug 2009, 11:17)=-=- Dependency deleted: 39 no longer depends on 45 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 45 -=-=(Psergey - Sat, 15 Aug 2009, 23:40)=-=- Title modified. --- /tmp/wklog.45.old.17603 2009-08-15 23:40:38.000000000 +0300 +++ /tmp/wklog.45.new.17603 2009-08-15 23:40:38.000000000 +0300 @@ -1 +1 @@ -Add a mysqlbinlog option to produce siccint output +Add a mysqlbinlog option to produce succint output DESCRIPTION: Add a mysqlbinlog option to produce the most succinct output, without any comments or other statements that are not needed to apply binlog correctly. This will be different from --short-form option. That option causes mysqlbinlog not to print RBR events, i.e. the output is not supposed to be applied. There is also option to have --short-form do what this WL requires, see MySQL BUG#18337. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Add a mysqlbinlog option to produce succint output (45)
by worklog-noreply＠askmonty.org 17 Aug '09

17 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add a mysqlbinlog option to produce succint output CREATION DATE..: Sat, 15 Aug 2009, 23:40 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 45 (http://askmonty.org/worklog/?tid=45) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Mon, 17 Aug 2009, 13:58)=-=- High Level Description modified. --- /tmp/wklog.45.old.10768 2009-08-17 13:58:07.000000000 +0300 +++ /tmp/wklog.45.new.10768 2009-08-17 13:58:07.000000000 +0300 @@ -4,3 +4,5 @@ This will be different from --short-form option. That option causes mysqlbinlog not to print RBR events, i.e. the output is not supposed to be applied. +There is also option to have --short-form do what this WL requires, see MySQL +BUG#18337. -=-=(Guest - Mon, 17 Aug 2009, 11:17)=-=- Dependency created: 39 now depends on 45 -=-=(Guest - Mon, 17 Aug 2009, 11:17)=-=- Dependency deleted: 39 no longer depends on 45 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 45 -=-=(Psergey - Sat, 15 Aug 2009, 23:40)=-=- Title modified. --- /tmp/wklog.45.old.17603 2009-08-15 23:40:38.000000000 +0300 +++ /tmp/wklog.45.new.17603 2009-08-15 23:40:38.000000000 +0300 @@ -1 +1 @@ -Add a mysqlbinlog option to produce siccint output +Add a mysqlbinlog option to produce succint output DESCRIPTION: Add a mysqlbinlog option to produce the most succinct output, without any comments or other statements that are not needed to apply binlog correctly. This will be different from --short-form option. That option causes mysqlbinlog not to print RBR events, i.e. the output is not supposed to be applied. There is also option to have --short-form do what this WL requires, see MySQL BUG#18337. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Knielsen): Add a mysqlbinlog option to filter certain kinds of statements (41)
by worklog-noreply＠askmonty.org 17 Aug '09

17 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add a mysqlbinlog option to filter certain kinds of statements CREATION DATE..: Mon, 10 Aug 2009, 15:30 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Client-BackLog TASK ID........: 41 (http://askmonty.org/worklog/?tid=41) VERSION........: Benchmarks-3.0 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Mon, 17 Aug 2009, 13:56)=-=- High-Level Specification modified. --- /tmp/wklog.41.old.10632 2009-08-17 13:56:52.000000000 +0300 +++ /tmp/wklog.41.new.10632 2009-08-17 13:56:52.000000000 +0300 @@ -14,3 +14,10 @@ - Remove all pre-space - Compare the string case-insensitively - etc + +Option 3: + +Server-side support for ignoring certain statements: + + SET SESSION ignored_statements="alter table, analyze table, ..."; + -=-=(Knielsen - Fri, 14 Aug 2009, 14:17)=-=- High-Level Specification modified. --- /tmp/wklog.41.old.6963 2009-08-14 14:17:32.000000000 +0300 +++ /tmp/wklog.41.new.6963 2009-08-14 14:17:32.000000000 +0300 @@ -1,6 +1,11 @@ The implementation will depend on design choices made in WL#40: -- If we decide to parse the statement, SQL-verb filtering will be trivial -- If we decide not to parse the statement, we still can reliably distinguish the + +Option 1: + +If we decide to parse the statement, SQL-verb filtering will be trivial + +Option 2: +If we decide not to parse the statement, we still can reliably distinguish the statement by matching the first characters against a set of patterns. If we chose the second, we'll have to perform certain normalization before -=-=(Psergey - Mon, 10 Aug 2009, 15:47)=-=- High-Level Specification modified. --- /tmp/wklog.41.old.13282 2009-08-10 15:47:13.000000000 +0300 +++ /tmp/wklog.41.new.13282 2009-08-10 15:47:13.000000000 +0300 @@ -2,3 +2,10 @@ - If we decide to parse the statement, SQL-verb filtering will be trivial - If we decide not to parse the statement, we still can reliably distinguish the statement by matching the first characters against a set of patterns. + +If we chose the second, we'll have to perform certain normalization before +matching the patterns: + - Remove all comments from the command + - Remove all pre-space + - Compare the string case-insensitively + - etc -=-=(Psergey - Mon, 10 Aug 2009, 15:35)=-=- High-Level Specification modified. --- /tmp/wklog.41.old.12689 2009-08-10 15:35:04.000000000 +0300 +++ /tmp/wklog.41.new.12689 2009-08-10 15:35:04.000000000 +0300 @@ -1 +1,4 @@ - +The implementation will depend on design choices made in WL#40: +- If we decide to parse the statement, SQL-verb filtering will be trivial +- If we decide not to parse the statement, we still can reliably distinguish the +statement by matching the first characters against a set of patterns. -=-=(Psergey - Mon, 10 Aug 2009, 15:31)=-=- Dependency created: 39 now depends on 41 DESCRIPTION: Add a mysqlbinlog option to filter certain kinds of statements, i.e. (syntax subject to discussion): mysqlbinlog --exclude='alter table,drop table,alter database,...' HIGH-LEVEL SPECIFICATION: The implementation will depend on design choices made in WL#40: Option 1: If we decide to parse the statement, SQL-verb filtering will be trivial Option 2: If we decide not to parse the statement, we still can reliably distinguish the statement by matching the first characters against a set of patterns. If we chose the second, we'll have to perform certain normalization before matching the patterns: - Remove all comments from the command - Remove all pre-space - Compare the string case-insensitively - etc Option 3: Server-side support for ignoring certain statements: SET SESSION ignored_statements="alter table, analyze table, ..."; ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Knielsen): Add a mysqlbinlog option to filter certain kinds of statements (41)
by worklog-noreply＠askmonty.org 17 Aug '09

17 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add a mysqlbinlog option to filter certain kinds of statements CREATION DATE..: Mon, 10 Aug 2009, 15:30 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Client-BackLog TASK ID........: 41 (http://askmonty.org/worklog/?tid=41) VERSION........: Benchmarks-3.0 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Mon, 17 Aug 2009, 13:56)=-=- High-Level Specification modified. --- /tmp/wklog.41.old.10632 2009-08-17 13:56:52.000000000 +0300 +++ /tmp/wklog.41.new.10632 2009-08-17 13:56:52.000000000 +0300 @@ -14,3 +14,10 @@ - Remove all pre-space - Compare the string case-insensitively - etc + +Option 3: + +Server-side support for ignoring certain statements: + + SET SESSION ignored_statements="alter table, analyze table, ..."; + -=-=(Knielsen - Fri, 14 Aug 2009, 14:17)=-=- High-Level Specification modified. --- /tmp/wklog.41.old.6963 2009-08-14 14:17:32.000000000 +0300 +++ /tmp/wklog.41.new.6963 2009-08-14 14:17:32.000000000 +0300 @@ -1,6 +1,11 @@ The implementation will depend on design choices made in WL#40: -- If we decide to parse the statement, SQL-verb filtering will be trivial -- If we decide not to parse the statement, we still can reliably distinguish the + +Option 1: + +If we decide to parse the statement, SQL-verb filtering will be trivial + +Option 2: +If we decide not to parse the statement, we still can reliably distinguish the statement by matching the first characters against a set of patterns. If we chose the second, we'll have to perform certain normalization before -=-=(Psergey - Mon, 10 Aug 2009, 15:47)=-=- High-Level Specification modified. --- /tmp/wklog.41.old.13282 2009-08-10 15:47:13.000000000 +0300 +++ /tmp/wklog.41.new.13282 2009-08-10 15:47:13.000000000 +0300 @@ -2,3 +2,10 @@ - If we decide to parse the statement, SQL-verb filtering will be trivial - If we decide not to parse the statement, we still can reliably distinguish the statement by matching the first characters against a set of patterns. + +If we chose the second, we'll have to perform certain normalization before +matching the patterns: + - Remove all comments from the command + - Remove all pre-space + - Compare the string case-insensitively + - etc -=-=(Psergey - Mon, 10 Aug 2009, 15:35)=-=- High-Level Specification modified. --- /tmp/wklog.41.old.12689 2009-08-10 15:35:04.000000000 +0300 +++ /tmp/wklog.41.new.12689 2009-08-10 15:35:04.000000000 +0300 @@ -1 +1,4 @@ - +The implementation will depend on design choices made in WL#40: +- If we decide to parse the statement, SQL-verb filtering will be trivial +- If we decide not to parse the statement, we still can reliably distinguish the +statement by matching the first characters against a set of patterns. -=-=(Psergey - Mon, 10 Aug 2009, 15:31)=-=- Dependency created: 39 now depends on 41 DESCRIPTION: Add a mysqlbinlog option to filter certain kinds of statements, i.e. (syntax subject to discussion): mysqlbinlog --exclude='alter table,drop table,alter database,...' HIGH-LEVEL SPECIFICATION: The implementation will depend on design choices made in WL#40: Option 1: If we decide to parse the statement, SQL-verb filtering will be trivial Option 2: If we decide not to parse the statement, we still can reliably distinguish the statement by matching the first characters against a set of patterns. If we chose the second, we'll have to perform certain normalization before matching the patterns: - Remove all comments from the command - Remove all pre-space - Compare the string case-insensitively - etc Option 3: Server-side support for ignoring certain statements: SET SESSION ignored_statements="alter table, analyze table, ..."; ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Knielsen): Add a mysqlbinlog option to change the used database (36)
by worklog-noreply＠askmonty.org 17 Aug '09

17 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add a mysqlbinlog option to change the used database CREATION DATE..: Fri, 07 Aug 2009, 14:57 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 36 (http://askmonty.org/worklog/?tid=36) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Mon, 17 Aug 2009, 12:44)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.7834 2009-08-17 12:44:17.000000000 +0300 +++ /tmp/wklog.36.new.7834 2009-08-17 12:44:17.000000000 +0300 @@ -13,7 +13,9 @@ statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). -See also MySQL BUG#42941. +See also MySQL BUG#42941. Note this bug is fixed in MySQL 5.1.37, which is not +merged into MariaDB at the time of writing, but planned to be merged before +release. What we could do ---------------- -=-=(Guest - Sun, 16 Aug 2009, 17:11)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.27162 2009-08-16 17:11:12.000000000 +0300 +++ /tmp/wklog.36.new.27162 2009-08-16 17:11:12.000000000 +0300 @@ -13,6 +13,8 @@ statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). +See also MySQL BUG#42941. + What we could do ---------------- -=-=(Psergey - Mon, 10 Aug 2009, 15:41)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.13035 2009-08-10 15:41:51.000000000 +0300 +++ /tmp/wklog.36.new.13035 2009-08-10 15:41:51.000000000 +0300 @@ -1,5 +1,7 @@ Context ------- +(See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global +overview) At the moment, the server has a replication slave option --replicate-rewrite-db="from->to" -=-=(Guest - Mon, 10 Aug 2009, 11:12)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.6580 2009-08-10 11:12:36.000000000 +0300 +++ /tmp/wklog.36.new.6580 2009-08-10 11:12:36.000000000 +0300 @@ -1,4 +1,3 @@ - Context ------- At the moment, the server has a replication slave option @@ -67,6 +66,6 @@ It will be possible to do the rewrites either on the slave ( --replicate-rewrite-db will work for all kinds of statements), or in -mysqlbinlog (adding a comment is easy and doesn't require use to parse the -statement). +mysqlbinlog (adding a comment is easy and doesn't require mysqlbinlog to +parse the statement). -=-=(Psergey - Sun, 09 Aug 2009, 23:53)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.13425 2009-08-09 23:53:54.000000000 +0300 +++ /tmp/wklog.36.new.13425 2009-08-09 23:53:54.000000000 +0300 @@ -1 +1,72 @@ +Context +------- +At the moment, the server has a replication slave option + + --replicate-rewrite-db="from->to" + +the option affects +- Table_map_log_event (all RBR events) +- Load_log_event (LOAD DATA) +- Query_log_event (SBR-based updates, with the usual assumption that the + statement refers to tables in current database, so that changing the current + database will make the statement to work on a table in a different database). + +What we could do +---------------- + +Option1: make mysqlbinlog accept --replicate-rewrite-db option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Make mysqlbinlog accept --replicate-rewrite-db options and process them to the +same extent as replication slave would process --replicate-rewrite-db option. + + +Option2: Add database-agnostic RBR events and --strip-db option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Right now RBR events require a databasename. It is not possible to have RBR +event stream that won't mention which database the events are for. When I +tried to use debugger and specify empty database name, attempt to apply the +binlog resulted in this error: + +090809 17:38:44 [ERROR] Slave SQL: Error 'Table '.tablename' doesn't exist' on +opening tables, + +We could do as follows: +- Make the server interpret empty database name in RBR event (i.e. in a + Table_map_log_event) as "use current database". Binlog slave thread + probably should not allow such events as it doesn't have a natural current + database. +- Add a mysqlbinlog --strip-db option that would + = not produce any "USE dbname" statements + = change databasename for all RBR events to be empty + +That way, mysqlbinlog output will be database-agnostic and apply to the +current database. +(this will have the usual limitations that we assume that all statements in +the binlog refer to the current database). + +Option3: Enhance database rewrite +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +If there is a need to support database change for statements that use +dbname.tablename notation and are replicated as statements (i.e. are DDL +statements and/or DML statements that are binlogged as statements), +then that could be supported as follows: + +- Make the server's parser recognize special form of comments + + /* !database-alias(oldname,newname) */ + + and save the mapping somewhere + +- Put the hooks in table open and name resolution code to use the saved + mapping. + + +Once we've done the above, it will be easy to perform a complete, +no-compromise or restrictions database name change in binary log. + +It will be possible to do the rewrites either on the slave ( +--replicate-rewrite-db will work for all kinds of statements), or in +mysqlbinlog (adding a comment is easy and doesn't require use to parse the +statement). + -=-=(Psergey - Sun, 09 Aug 2009, 12:27)=-=- Dependency created: 39 now depends on 36 -=-=(Psergey - Fri, 07 Aug 2009, 14:57)=-=- Title modified. --- /tmp/wklog.36.old.14687 2009-08-07 14:57:49.000000000 +0300 +++ /tmp/wklog.36.new.14687 2009-08-07 14:57:49.000000000 +0300 @@ -1 +1 @@ -Add a mysqlbinlog option to change the database +Add a mysqlbinlog option to change the used database DESCRIPTION: Sometimes there is a need to take a binary log and apply it to a database with a different name than the original name of the database on binlog producer. If one is using statement-based replication, he can achieve this by grepping out "USE dbname" statements out of the output of mysqlbinlog(*). With row-based replication this is no longer possible, as database name is encoded within the the BINLOG '....' statement. This task is about adding an option to mysqlbinlog that would allow to change the names of used databases in both RBR and SBR events. (*) this implies that all statements refer to tables in the current database, doesn't catch updates made inside stored functions and so forth, but still works for a practially-important subset of cases. HIGH-LEVEL SPECIFICATION: Context ------- (See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global overview) At the moment, the server has a replication slave option --replicate-rewrite-db="from->to" the option affects - Table_map_log_event (all RBR events) - Load_log_event (LOAD DATA) - Query_log_event (SBR-based updates, with the usual assumption that the statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). See also MySQL BUG#42941. Note this bug is fixed in MySQL 5.1.37, which is not merged into MariaDB at the time of writing, but planned to be merged before release. What we could do ---------------- Option1: make mysqlbinlog accept --replicate-rewrite-db option ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Make mysqlbinlog accept --replicate-rewrite-db options and process them to the same extent as replication slave would process --replicate-rewrite-db option. Option2: Add database-agnostic RBR events and --strip-db option ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Right now RBR events require a databasename. It is not possible to have RBR event stream that won't mention which database the events are for. When I tried to use debugger and specify empty database name, attempt to apply the binlog resulted in this error: 090809 17:38:44 [ERROR] Slave SQL: Error 'Table '.tablename' doesn't exist' on opening tables, We could do as follows: - Make the server interpret empty database name in RBR event (i.e. in a Table_map_log_event) as "use current database". Binlog slave thread probably should not allow such events as it doesn't have a natural current database. - Add a mysqlbinlog --strip-db option that would = not produce any "USE dbname" statements = change databasename for all RBR events to be empty That way, mysqlbinlog output will be database-agnostic and apply to the current database. (this will have the usual limitations that we assume that all statements in the binlog refer to the current database). Option3: Enhance database rewrite ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If there is a need to support database change for statements that use dbname.tablename notation and are replicated as statements (i.e. are DDL statements and/or DML statements that are binlogged as statements), then that could be supported as follows: - Make the server's parser recognize special form of comments /* !database-alias(oldname,newname) */ and save the mapping somewhere - Put the hooks in table open and name resolution code to use the saved mapping. Once we've done the above, it will be easy to perform a complete, no-compromise or restrictions database name change in binary log. It will be possible to do the rewrites either on the slave ( --replicate-rewrite-db will work for all kinds of statements), or in mysqlbinlog (adding a comment is easy and doesn't require mysqlbinlog to parse the statement). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Knielsen): Add a mysqlbinlog option to change the used database (36)
by worklog-noreply＠askmonty.org 17 Aug '09

17 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add a mysqlbinlog option to change the used database CREATION DATE..: Fri, 07 Aug 2009, 14:57 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 36 (http://askmonty.org/worklog/?tid=36) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Mon, 17 Aug 2009, 12:44)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.7834 2009-08-17 12:44:17.000000000 +0300 +++ /tmp/wklog.36.new.7834 2009-08-17 12:44:17.000000000 +0300 @@ -13,7 +13,9 @@ statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). -See also MySQL BUG#42941. +See also MySQL BUG#42941. Note this bug is fixed in MySQL 5.1.37, which is not +merged into MariaDB at the time of writing, but planned to be merged before +release. What we could do ---------------- -=-=(Guest - Sun, 16 Aug 2009, 17:11)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.27162 2009-08-16 17:11:12.000000000 +0300 +++ /tmp/wklog.36.new.27162 2009-08-16 17:11:12.000000000 +0300 @@ -13,6 +13,8 @@ statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). +See also MySQL BUG#42941. + What we could do ---------------- -=-=(Psergey - Mon, 10 Aug 2009, 15:41)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.13035 2009-08-10 15:41:51.000000000 +0300 +++ /tmp/wklog.36.new.13035 2009-08-10 15:41:51.000000000 +0300 @@ -1,5 +1,7 @@ Context ------- +(See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global +overview) At the moment, the server has a replication slave option --replicate-rewrite-db="from->to" -=-=(Guest - Mon, 10 Aug 2009, 11:12)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.6580 2009-08-10 11:12:36.000000000 +0300 +++ /tmp/wklog.36.new.6580 2009-08-10 11:12:36.000000000 +0300 @@ -1,4 +1,3 @@ - Context ------- At the moment, the server has a replication slave option @@ -67,6 +66,6 @@ It will be possible to do the rewrites either on the slave ( --replicate-rewrite-db will work for all kinds of statements), or in -mysqlbinlog (adding a comment is easy and doesn't require use to parse the -statement). +mysqlbinlog (adding a comment is easy and doesn't require mysqlbinlog to +parse the statement). -=-=(Psergey - Sun, 09 Aug 2009, 23:53)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.13425 2009-08-09 23:53:54.000000000 +0300 +++ /tmp/wklog.36.new.13425 2009-08-09 23:53:54.000000000 +0300 @@ -1 +1,72 @@ +Context +------- +At the moment, the server has a replication slave option + + --replicate-rewrite-db="from->to" + +the option affects +- Table_map_log_event (all RBR events) +- Load_log_event (LOAD DATA) +- Query_log_event (SBR-based updates, with the usual assumption that the + statement refers to tables in current database, so that changing the current + database will make the statement to work on a table in a different database). + +What we could do +---------------- + +Option1: make mysqlbinlog accept --replicate-rewrite-db option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Make mysqlbinlog accept --replicate-rewrite-db options and process them to the +same extent as replication slave would process --replicate-rewrite-db option. + + +Option2: Add database-agnostic RBR events and --strip-db option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Right now RBR events require a databasename. It is not possible to have RBR +event stream that won't mention which database the events are for. When I +tried to use debugger and specify empty database name, attempt to apply the +binlog resulted in this error: + +090809 17:38:44 [ERROR] Slave SQL: Error 'Table '.tablename' doesn't exist' on +opening tables, + +We could do as follows: +- Make the server interpret empty database name in RBR event (i.e. in a + Table_map_log_event) as "use current database". Binlog slave thread + probably should not allow such events as it doesn't have a natural current + database. +- Add a mysqlbinlog --strip-db option that would + = not produce any "USE dbname" statements + = change databasename for all RBR events to be empty + +That way, mysqlbinlog output will be database-agnostic and apply to the +current database. +(this will have the usual limitations that we assume that all statements in +the binlog refer to the current database). + +Option3: Enhance database rewrite +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +If there is a need to support database change for statements that use +dbname.tablename notation and are replicated as statements (i.e. are DDL +statements and/or DML statements that are binlogged as statements), +then that could be supported as follows: + +- Make the server's parser recognize special form of comments + + /* !database-alias(oldname,newname) */ + + and save the mapping somewhere + +- Put the hooks in table open and name resolution code to use the saved + mapping. + + +Once we've done the above, it will be easy to perform a complete, +no-compromise or restrictions database name change in binary log. + +It will be possible to do the rewrites either on the slave ( +--replicate-rewrite-db will work for all kinds of statements), or in +mysqlbinlog (adding a comment is easy and doesn't require use to parse the +statement). + -=-=(Psergey - Sun, 09 Aug 2009, 12:27)=-=- Dependency created: 39 now depends on 36 -=-=(Psergey - Fri, 07 Aug 2009, 14:57)=-=- Title modified. --- /tmp/wklog.36.old.14687 2009-08-07 14:57:49.000000000 +0300 +++ /tmp/wklog.36.new.14687 2009-08-07 14:57:49.000000000 +0300 @@ -1 +1 @@ -Add a mysqlbinlog option to change the database +Add a mysqlbinlog option to change the used database DESCRIPTION: Sometimes there is a need to take a binary log and apply it to a database with a different name than the original name of the database on binlog producer. If one is using statement-based replication, he can achieve this by grepping out "USE dbname" statements out of the output of mysqlbinlog(*). With row-based replication this is no longer possible, as database name is encoded within the the BINLOG '....' statement. This task is about adding an option to mysqlbinlog that would allow to change the names of used databases in both RBR and SBR events. (*) this implies that all statements refer to tables in the current database, doesn't catch updates made inside stored functions and so forth, but still works for a practially-important subset of cases. HIGH-LEVEL SPECIFICATION: Context ------- (See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global overview) At the moment, the server has a replication slave option --replicate-rewrite-db="from->to" the option affects - Table_map_log_event (all RBR events) - Load_log_event (LOAD DATA) - Query_log_event (SBR-based updates, with the usual assumption that the statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). See also MySQL BUG#42941. Note this bug is fixed in MySQL 5.1.37, which is not merged into MariaDB at the time of writing, but planned to be merged before release. What we could do ---------------- Option1: make mysqlbinlog accept --replicate-rewrite-db option ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Make mysqlbinlog accept --replicate-rewrite-db options and process them to the same extent as replication slave would process --replicate-rewrite-db option. Option2: Add database-agnostic RBR events and --strip-db option ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Right now RBR events require a databasename. It is not possible to have RBR event stream that won't mention which database the events are for. When I tried to use debugger and specify empty database name, attempt to apply the binlog resulted in this error: 090809 17:38:44 [ERROR] Slave SQL: Error 'Table '.tablename' doesn't exist' on opening tables, We could do as follows: - Make the server interpret empty database name in RBR event (i.e. in a Table_map_log_event) as "use current database". Binlog slave thread probably should not allow such events as it doesn't have a natural current database. - Add a mysqlbinlog --strip-db option that would = not produce any "USE dbname" statements = change databasename for all RBR events to be empty That way, mysqlbinlog output will be database-agnostic and apply to the current database. (this will have the usual limitations that we assume that all statements in the binlog refer to the current database). Option3: Enhance database rewrite ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If there is a need to support database change for statements that use dbname.tablename notation and are replicated as statements (i.e. are DDL statements and/or DML statements that are binlogged as statements), then that could be supported as follows: - Make the server's parser recognize special form of comments /* !database-alias(oldname,newname) */ and save the mapping somewhere - Put the hooks in table open and name resolution code to use the saved mapping. Once we've done the above, it will be easy to perform a complete, no-compromise or restrictions database name change in binary log. It will be possible to do the rewrites either on the slave ( --replicate-rewrite-db will work for all kinds of statements), or in mysqlbinlog (adding a comment is easy and doesn't require mysqlbinlog to parse the statement). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Knielsen): Change BINLOG statement syntax to be human-readable (46)
by worklog-noreply＠askmonty.org 17 Aug '09

17 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Change BINLOG statement syntax to be human-readable CREATION DATE..: Sat, 15 Aug 2009, 23:42 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 46 (http://askmonty.org/worklog/?tid=46) VERSION........: WorkLog-3.4 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Mon, 17 Aug 2009, 11:38)=-=- High-Level Specification modified. --- /tmp/wklog.46.old.4940 2009-08-17 11:38:49.000000000 +0300 +++ /tmp/wklog.46.new.4940 2009-08-17 11:38:49.000000000 +0300 @@ -16,8 +16,8 @@ Feedback and other suggestions ------------------------------ -* What is the need for WITH TIMESTAMP part? Can't one use a separate - SET TIMESTAMP statement? +* TIMESTAMP part is better omitted and replaced with separate SET TIMESTAMP +statement for consistency with other events. * mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something that's close to readable SQL. Can we make it to be regular parseable SQL? @@ -25,6 +25,7 @@ - A stream of SQL statements will be slower to run than BINLOG statements (due to locking, table open/close, etc). (TODO: is it really slower? we haven't checked). + One problem is that we do not have column names available in the binary log. * When SBR replication is used and the statements refer to the current database (a common scenario), one can use awk to filter out updates made in certain -=-=(Psergey - Sun, 16 Aug 2009, 11:30)=-=- High-Level Specification modified. --- /tmp/wklog.46.old.13453 2009-08-16 11:30:06.000000000 +0300 +++ /tmp/wklog.46.new.13453 2009-08-16 11:30:06.000000000 +0300 @@ -26,3 +26,7 @@ (due to locking, table open/close, etc). (TODO: is it really slower? we haven't checked). +* When SBR replication is used and the statements refer to the current database + (a common scenario), one can use awk to filter out updates made in certain + databases. The proposed syntax doesn't allow to perform equivalent filtering? + -=-=(Psergey - Sun, 16 Aug 2009, 11:13)=-=- High Level Description modified. --- /tmp/wklog.46.old.12747 2009-08-16 11:13:54.000000000 +0300 +++ /tmp/wklog.46.new.12747 2009-08-16 11:13:54.000000000 +0300 @@ -6,4 +6,4 @@ This WL task is about making BINLOG statements to be human-readable (either as an option or by default -The approach of this WL is to some extent an alternative to WL#38, WL#40, WL41. +The approach of this WL is to some extent an alternative to WL#38, WL#40, WL#41. -=-=(Psergey - Sun, 16 Aug 2009, 11:13)=-=- High Level Description modified. --- /tmp/wklog.46.old.12717 2009-08-16 11:13:40.000000000 +0300 +++ /tmp/wklog.46.new.12717 2009-08-16 11:13:40.000000000 +0300 @@ -5,3 +5,5 @@ This WL task is about making BINLOG statements to be human-readable (either as an option or by default + +The approach of this WL is to some extent an alternative to WL#38, WL#40, WL41. -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency deleted: 48 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency created: 48 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency deleted: 39 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sat, 15 Aug 2009, 23:43)=-=- High-Level Specification modified. --- /tmp/wklog.46.old.17742 2009-08-15 23:43:09.000000000 +0300 +++ /tmp/wklog.46.new.17742 2009-08-15 23:43:09.000000000 +0300 @@ -1 +1,28 @@ +Suggestion 1 +------------ +Original syntax suggestion by Kristian: + + BINLOG + WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 + TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 + TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 + WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 + UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), + FROM ('a') TO ('b') FLAGS 0x0 + DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; + + This is basically a dump of what is stored in the events, and would be an + alternative to BINLOG 'gwWEShMBAA...'. + +Feedback and other suggestions +------------------------------ +* What is the need for WITH TIMESTAMP part? Can't one use a separate + SET TIMESTAMP statement? + +* mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something + that's close to readable SQL. Can we make it to be regular parseable SQL? + + This will be syntax that's familiar to our parser and to the users + - A stream of SQL statements will be slower to run than BINLOG statements + (due to locking, table open/close, etc). (TODO: is it really slower? we + haven't checked). DESCRIPTION: One of great things about mysqlbinlog was that its output was human-readable SQL, so it was possible to edit it manually or with help of scripts. With RBR events and BINLOG 'DpiGShMBAAAALQAAADcBAA...' statements this is no longer the case. This WL task is about making BINLOG statements to be human-readable (either as an option or by default The approach of this WL is to some extent an alternative to WL#38, WL#40, WL#41. HIGH-LEVEL SPECIFICATION: Suggestion 1 ------------ Original syntax suggestion by Kristian: BINLOG WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), FROM ('a') TO ('b') FLAGS 0x0 DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; This is basically a dump of what is stored in the events, and would be an alternative to BINLOG 'gwWEShMBAA...'. Feedback and other suggestions ------------------------------ * TIMESTAMP part is better omitted and replaced with separate SET TIMESTAMP statement for consistency with other events. * mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something that's close to readable SQL. Can we make it to be regular parseable SQL? + This will be syntax that's familiar to our parser and to the users - A stream of SQL statements will be slower to run than BINLOG statements (due to locking, table open/close, etc). (TODO: is it really slower? we haven't checked). One problem is that we do not have column names available in the binary log. * When SBR replication is used and the statements refer to the current database (a common scenario), one can use awk to filter out updates made in certain databases. The proposed syntax doesn't allow to perform equivalent filtering? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Knielsen): Change BINLOG statement syntax to be human-readable (46)
by worklog-noreply＠askmonty.org 17 Aug '09

17 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Change BINLOG statement syntax to be human-readable CREATION DATE..: Sat, 15 Aug 2009, 23:42 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 46 (http://askmonty.org/worklog/?tid=46) VERSION........: WorkLog-3.4 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Mon, 17 Aug 2009, 11:38)=-=- High-Level Specification modified. --- /tmp/wklog.46.old.4940 2009-08-17 11:38:49.000000000 +0300 +++ /tmp/wklog.46.new.4940 2009-08-17 11:38:49.000000000 +0300 @@ -16,8 +16,8 @@ Feedback and other suggestions ------------------------------ -* What is the need for WITH TIMESTAMP part? Can't one use a separate - SET TIMESTAMP statement? +* TIMESTAMP part is better omitted and replaced with separate SET TIMESTAMP +statement for consistency with other events. * mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something that's close to readable SQL. Can we make it to be regular parseable SQL? @@ -25,6 +25,7 @@ - A stream of SQL statements will be slower to run than BINLOG statements (due to locking, table open/close, etc). (TODO: is it really slower? we haven't checked). + One problem is that we do not have column names available in the binary log. * When SBR replication is used and the statements refer to the current database (a common scenario), one can use awk to filter out updates made in certain -=-=(Psergey - Sun, 16 Aug 2009, 11:30)=-=- High-Level Specification modified. --- /tmp/wklog.46.old.13453 2009-08-16 11:30:06.000000000 +0300 +++ /tmp/wklog.46.new.13453 2009-08-16 11:30:06.000000000 +0300 @@ -26,3 +26,7 @@ (due to locking, table open/close, etc). (TODO: is it really slower? we haven't checked). +* When SBR replication is used and the statements refer to the current database + (a common scenario), one can use awk to filter out updates made in certain + databases. The proposed syntax doesn't allow to perform equivalent filtering? + -=-=(Psergey - Sun, 16 Aug 2009, 11:13)=-=- High Level Description modified. --- /tmp/wklog.46.old.12747 2009-08-16 11:13:54.000000000 +0300 +++ /tmp/wklog.46.new.12747 2009-08-16 11:13:54.000000000 +0300 @@ -6,4 +6,4 @@ This WL task is about making BINLOG statements to be human-readable (either as an option or by default -The approach of this WL is to some extent an alternative to WL#38, WL#40, WL41. +The approach of this WL is to some extent an alternative to WL#38, WL#40, WL#41. -=-=(Psergey - Sun, 16 Aug 2009, 11:13)=-=- High Level Description modified. --- /tmp/wklog.46.old.12717 2009-08-16 11:13:40.000000000 +0300 +++ /tmp/wklog.46.new.12717 2009-08-16 11:13:40.000000000 +0300 @@ -5,3 +5,5 @@ This WL task is about making BINLOG statements to be human-readable (either as an option or by default + +The approach of this WL is to some extent an alternative to WL#38, WL#40, WL41. -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency deleted: 48 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency created: 48 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency deleted: 39 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sat, 15 Aug 2009, 23:43)=-=- High-Level Specification modified. --- /tmp/wklog.46.old.17742 2009-08-15 23:43:09.000000000 +0300 +++ /tmp/wklog.46.new.17742 2009-08-15 23:43:09.000000000 +0300 @@ -1 +1,28 @@ +Suggestion 1 +------------ +Original syntax suggestion by Kristian: + + BINLOG + WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 + TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 + TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 + WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 + UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), + FROM ('a') TO ('b') FLAGS 0x0 + DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; + + This is basically a dump of what is stored in the events, and would be an + alternative to BINLOG 'gwWEShMBAA...'. + +Feedback and other suggestions +------------------------------ +* What is the need for WITH TIMESTAMP part? Can't one use a separate + SET TIMESTAMP statement? + +* mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something + that's close to readable SQL. Can we make it to be regular parseable SQL? + + This will be syntax that's familiar to our parser and to the users + - A stream of SQL statements will be slower to run than BINLOG statements + (due to locking, table open/close, etc). (TODO: is it really slower? we + haven't checked). DESCRIPTION: One of great things about mysqlbinlog was that its output was human-readable SQL, so it was possible to edit it manually or with help of scripts. With RBR events and BINLOG 'DpiGShMBAAAALQAAADcBAA...' statements this is no longer the case. This WL task is about making BINLOG statements to be human-readable (either as an option or by default The approach of this WL is to some extent an alternative to WL#38, WL#40, WL#41. HIGH-LEVEL SPECIFICATION: Suggestion 1 ------------ Original syntax suggestion by Kristian: BINLOG WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), FROM ('a') TO ('b') FLAGS 0x0 DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; This is basically a dump of what is stored in the events, and would be an alternative to BINLOG 'gwWEShMBAA...'. Feedback and other suggestions ------------------------------ * TIMESTAMP part is better omitted and replaced with separate SET TIMESTAMP statement for consistency with other events. * mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something that's close to readable SQL. Can we make it to be regular parseable SQL? + This will be syntax that's familiar to our parser and to the users - A stream of SQL statements will be slower to run than BINLOG statements (due to locking, table open/close, etc). (TODO: is it really slower? we haven't checked). One problem is that we do not have column names available in the binary log. * When SBR replication is used and the statements refer to the current database (a common scenario), one can use awk to filter out updates made in certain databases. The proposed syntax doesn't allow to perform equivalent filtering? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Progress (by Knielsen): improving mysqlbinlog output and doing rename (39)
by worklog-noreply＠askmonty.org 17 Aug '09

17 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: improving mysqlbinlog output and doing rename CREATION DATE..: Sun, 09 Aug 2009, 12:24 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Client-RawIdeaBin TASK ID........: 39 (http://askmonty.org/worklog/?tid=39) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 29 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Mon, 17 Aug 2009, 11:13)=-=- Research and architecture review. Worked 12 hours and estimate 0 hours remain (original estimate increased by 12 hours). -=-=(Knielsen - Mon, 17 Aug 2009, 11:13)=-=- Reported zero hours worked. Estimate unchanged. -=-=(Psergey - Sun, 16 Aug 2009, 12:07)=-=- Dependency created: 39 now depends on 49 -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency deleted: 39 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 47 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 45 -=-=(Guest - Fri, 14 Aug 2009, 15:52)=-=- Title modified. --- /tmp/wklog.39.old.11123 2009-08-14 15:52:29.000000000 +0300 +++ /tmp/wklog.39.new.11123 2009-08-14 15:52:29.000000000 +0300 @@ -1 +1 @@ -Replication tasks +improving mysqlbinlog output and doing rename -=-=(Guest - Mon, 10 Aug 2009, 16:32)=-=- Adding 1 hour for Monty's initial work on starting the architecture review. Worked 1 hour and estimate 0 hours remain (original estimate increased by 1 hour). ------------------------------------------------------------ -=-=(View All Progress Notes, 16 total)=-=- http://askmonty.org/worklog/index.pl?tid=39&nolimit=1 DESCRIPTION: A combine task for all replication tasks. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Progress (by Knielsen): improving mysqlbinlog output and doing rename (39)
by worklog-noreply＠askmonty.org 17 Aug '09

17 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: improving mysqlbinlog output and doing rename CREATION DATE..: Sun, 09 Aug 2009, 12:24 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Client-RawIdeaBin TASK ID........: 39 (http://askmonty.org/worklog/?tid=39) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 29 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Mon, 17 Aug 2009, 11:13)=-=- Research and architecture review. Worked 12 hours and estimate 0 hours remain (original estimate increased by 12 hours). -=-=(Knielsen - Mon, 17 Aug 2009, 11:13)=-=- Reported zero hours worked. Estimate unchanged. -=-=(Psergey - Sun, 16 Aug 2009, 12:07)=-=- Dependency created: 39 now depends on 49 -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency deleted: 39 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 47 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 45 -=-=(Guest - Fri, 14 Aug 2009, 15:52)=-=- Title modified. --- /tmp/wklog.39.old.11123 2009-08-14 15:52:29.000000000 +0300 +++ /tmp/wklog.39.new.11123 2009-08-14 15:52:29.000000000 +0300 @@ -1 +1 @@ -Replication tasks +improving mysqlbinlog output and doing rename -=-=(Guest - Mon, 10 Aug 2009, 16:32)=-=- Adding 1 hour for Monty's initial work on starting the architecture review. Worked 1 hour and estimate 0 hours remain (original estimate increased by 1 hour). ------------------------------------------------------------ -=-=(View All Progress Notes, 16 total)=-=- http://askmonty.org/worklog/index.pl?tid=39&nolimit=1 DESCRIPTION: A combine task for all replication tasks. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Progress (by Knielsen): improving mysqlbinlog output and doing rename (39)
by worklog-noreply＠askmonty.org 17 Aug '09

17 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: improving mysqlbinlog output and doing rename CREATION DATE..: Sun, 09 Aug 2009, 12:24 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Client-RawIdeaBin TASK ID........: 39 (http://askmonty.org/worklog/?tid=39) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 17 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Mon, 17 Aug 2009, 11:13)=-=- Reported zero hours worked. Estimate unchanged. -=-=(Psergey - Sun, 16 Aug 2009, 12:07)=-=- Dependency created: 39 now depends on 49 -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency deleted: 39 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 47 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 45 -=-=(Guest - Fri, 14 Aug 2009, 15:52)=-=- Title modified. --- /tmp/wklog.39.old.11123 2009-08-14 15:52:29.000000000 +0300 +++ /tmp/wklog.39.new.11123 2009-08-14 15:52:29.000000000 +0300 @@ -1 +1 @@ -Replication tasks +improving mysqlbinlog output and doing rename -=-=(Guest - Mon, 10 Aug 2009, 16:32)=-=- Adding 1 hour for Monty's initial work on starting the architecture review. Worked 1 hour and estimate 0 hours remain (original estimate increased by 1 hour). -=-=(Psergey - Mon, 10 Aug 2009, 15:59)=-=- Re-searched and added subtasks. Worked 16 hours and estimate 0 hours remain (original estimate increased by 16 hours). ------------------------------------------------------------ -=-=(View All Progress Notes, 15 total)=-=- http://askmonty.org/worklog/index.pl?tid=39&nolimit=1 DESCRIPTION: A combine task for all replication tasks. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Progress (by Knielsen): improving mysqlbinlog output and doing rename (39)
by worklog-noreply＠askmonty.org 17 Aug '09

17 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: improving mysqlbinlog output and doing rename CREATION DATE..: Sun, 09 Aug 2009, 12:24 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Client-RawIdeaBin TASK ID........: 39 (http://askmonty.org/worklog/?tid=39) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 17 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Mon, 17 Aug 2009, 11:13)=-=- Reported zero hours worked. Estimate unchanged. -=-=(Psergey - Sun, 16 Aug 2009, 12:07)=-=- Dependency created: 39 now depends on 49 -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency deleted: 39 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 47 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 45 -=-=(Guest - Fri, 14 Aug 2009, 15:52)=-=- Title modified. --- /tmp/wklog.39.old.11123 2009-08-14 15:52:29.000000000 +0300 +++ /tmp/wklog.39.new.11123 2009-08-14 15:52:29.000000000 +0300 @@ -1 +1 @@ -Replication tasks +improving mysqlbinlog output and doing rename -=-=(Guest - Mon, 10 Aug 2009, 16:32)=-=- Adding 1 hour for Monty's initial work on starting the architecture review. Worked 1 hour and estimate 0 hours remain (original estimate increased by 1 hour). -=-=(Psergey - Mon, 10 Aug 2009, 15:59)=-=- Re-searched and added subtasks. Worked 16 hours and estimate 0 hours remain (original estimate increased by 16 hours). ------------------------------------------------------------ -=-=(View All Progress Notes, 15 total)=-=- http://askmonty.org/worklog/index.pl?tid=39&nolimit=1 DESCRIPTION: A combine task for all replication tasks. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Rev 2730: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r10/
by Sergey Petrunya 16 Aug '09

16 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r10/ ------------------------------------------------------------ revno: 2730 revision-id: psergey(a)askmonty.org-20090816180159-z3lfkjpjfsm7zbp0 parent: psergey(a)askmonty.org-20090816143547-16hyle50tbt31xen committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Sun 2009-08-16 21:01:59 +0300 message: MWL#17: Table elimination - More comments === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-16 14:35:47 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-16 18:01:59 +0000 @@ -58,7 +58,7 @@ Table elimination is redone on every PS re-execution. - IMPLEMENTATION + TABLE ELIMINATION ALGORITHM As said above, we can remove inner side of an outer join if it is @@ -67,23 +67,59 @@ We check #1 by doing a recursive descent down the join->join_list while maintaining a union of used_tables() attribute of all expressions we've seen - "elsewhere". When we encounter an outer join, we check if the bitmap of - tables on its inner side has intersection with tables that are used - elsewhere. No intersection means that inner side of the outer join could + "elsewhere". When we encounter an outer join, we check if the bitmap of + tables on its inner side has an intersection with tables that are used + elsewhere. No intersection means that inner side of the outer join could potentially be eliminated. - #2 is checked using a concept of values and modules that indicate - dependencies between them. - - We start with - of certain values that functional dependencies between - them. There are two kinds of values: -*/ - -/* - A value - - functional dependencies between two kinds of entities: + In order to check #2, one needs to prove that inner side of an outer join + is functionally dependent on the outside. We prove dependency by proving + functional dependency of intermediate objects: + + - Inner side of outer join is functionally dependent when each of its tables + are functionally dependent. (We assume a table is functionally dependent + when its dependencies allow to uniquely identify one table record, or no + records). + + - Table is functionally dependent when it has got a unique key whose columns + are functionally dependent. + + - A column is functionally dependent when we could locate an AND-part of a + certain ON clause in form + + tblX.columnY= expr + + where expr is functionally-depdendent. + + Apparently the above rules can be applied recursively. Also, certain entities + depend on multiple other entities. We model this by a bipartite graph which + has two kinds of nodes: + + Value nodes: + - Table column values (each is a value of tblX.columnY) + - Table nodes (each node represents a table inside an eliminable join nest). + each value is either bound (i.e. functionally dependent) or not. + + Module nodes: + - Nodes representing tblX.colY=expr equalities. Equality node has + = incoming edges from columns used in expr + = outgoing edge to tblX.colY column. + - Nodes representing unique keys. Unique key has + = incoming edges from key component value nodes + = outgoing edge to key's table node + - Inner side of outer join node. Outer join node has + = incoming edges from table value nodes + = No outgoing edges. Once we reach it, we know we can eliminate the + outer join. + A module may depend on multiple values, and hence its primary attribute is + the number of its depedencies that are not bound. + + The algorithm starts with equality nodes that don't have any incoming edges + (their expressions are either constant or depend only on tables that are + outside of any outer joins) and proceeds to traverse dependency->dependant + edges until we've other traversed everything (TODO rephrase elaborate), or + we've reached the point where all outer join modules have zero unsatisfied + dependencies. */ class Value_dep;

1 0

Re: [Maria-developers] Sphinx Storage engine for MariaDB
by Andrew Aksyonoff 16 Aug '09

16 Aug '09

Hello Michael, Looks like I've overlooked this email back then. :( Peter pinged me about Sphinx vs Maria status recently and I just found it. Well, hopefully better late than never! Sunday, June 7, 2009, 1:18:30 PM, you wrote: MW> Andrew, what are the possible drawbacks you can see with having MW> Sphinx to be a part of MairaDB for a user that is not using Sphinx? Can't think of any. SphinxSE is a mere client and as such does not allocate any big RAM buffers or other resources. MW> I assume that if Sphinx is not enabled, it will not take any MW> resources. MW> If Sphinx is enabled but not used, what are the resorces it would use? Pretty much none, AFAIK. --- On an unrelated note, we're working on so called RT backend here, and it will allow all the normal CRUD operations in run time (as opposed to only having reads against a static fulltext index that we have now). When it's done it'll be also technically possible to integrate it too - and do so tighter by embedding the library instead of just talking to Sphinx searchd over network. Sphinx searchd can now talk MySQL protocol and supports basic SQL syntax. So for "just" full text tasks end users don't really need the integrated version. However it still might possibly be useful in certain use cases. To keep FT index in (better) sync with DB data, avoid overheads of double network roundtrips for additional processing, avoid hassles of keeping two connections and manually managing two open transactions, etc. So I wonder what'd be your opinion about the integration - whether it seems useful at all, and if yes, whether network client or embedded library route seems better. -- Best regards, Andrew mailto:shodan@shodan.ru

5 14

[Maria-developers] Rev 2729: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r10/
by Sergey Petrunya 16 Aug '09

16 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r10/ ------------------------------------------------------------ revno: 2729 revision-id: psergey(a)askmonty.org-20090816143547-16hyle50tbt31xen parent: psergey(a)askmonty.org-20090816124331-gd53m2alc0jb3ws4 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Sun 2009-08-16 17:35:47 +0300 message: MWL#17: Table elimination - Better comments - More OOM checks === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-16 12:43:31 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-16 14:35:47 +0000 @@ -19,6 +19,25 @@ /* OVERVIEW + This file contains table elimination module. The idea behind table + elimination is as follows: suppose we have a left join + + SELECT * FROM t1 LEFT JOIN + (t2 JOIN t3) ON t3.primary_key=t1.col AND + t4.primary_key=t2.col + such that + * columns of the inner tables are not used anywhere ouside the outer join + (not in WHERE, not in GROUP/ORDER BY clause, not in select list etc etc), + * inner side of the outer join is guaranteed to produce at most one matching + record combination for each record combination of outer tables. + + then the inner side of the outer join can be removed from the query, as it + will always produce only one record combination (either real or + null-complemented one) and we don't care about what that record combination + is. + + MODULE INTERFACE + The module has one entry point - eliminate_tables() function, which one needs to call (once) at some point before the join optimization. eliminate_tables() operates over the JOIN structures. Logically, it @@ -38,6 +57,50 @@ by EXPLAIN code to check if the subquery should be shown in EXPLAIN. Table elimination is redone on every PS re-execution. + + IMPLEMENTATION + + As said above, we can remove inner side of an outer join if it is + + 1. not referred to from any other parts of the query + 2. always produces one matching record combination. + + We check #1 by doing a recursive descent down the join->join_list while + maintaining a union of used_tables() attribute of all expressions we've seen + "elsewhere". When we encounter an outer join, we check if the bitmap of + tables on its inner side has intersection with tables that are used + elsewhere. No intersection means that inner side of the outer join could + potentially be eliminated. + + #2 is checked using a concept of values and modules that indicate + dependencies between them. + + We start with + of certain values that functional dependencies between + them. There are two kinds of values: +*/ + +/* + A value + + functional dependencies between two kinds of entities: +*/ + +class Value_dep; + class Field_value; + class Table_value; + + +class Module_dep; + class Equality_module; + class Outer_join_module; + class Key_module; + +class Table_elimination; + + +/* + A value. */ class Value_dep : public Sql_alloc @@ -55,13 +118,9 @@ Value_dep *next; }; -class Field_value; -class Table_value; -class Outer_join_module; -class Key_module; /* - A table field. There is only one such object for any tblX.fieldY + A table field value. There is exactly only one such object for any tblX.fieldY - the field epends on its table and equalities - expressions that use the field are its dependencies */ @@ -87,7 +146,8 @@ /* - A table. + A table value. There is one Table_value object for every table that can + potentially be eliminated. - table depends on any of its unique keys - has its fields and embedding outer join as dependency. */ @@ -221,6 +281,7 @@ MY_BITMAP expr_deps; }; + static bool build_eq_deps_for_cond(Table_elimination *te, Equality_module **fdeps, uint *and_level, Item *cond, @@ -244,6 +305,7 @@ #ifndef DBUG_OFF static void dbug_print_deps(Table_elimination *te); #endif + /*******************************************************************************************/ /* @@ -538,8 +600,8 @@ static bool add_eq_dep(Table_elimination *te, Equality_module **eq_dep, - uint and_level, Item_func *cond, - Item *left, Item *right, table_map usable_tables) + uint and_level, Item_func *cond, Item *left, Item *right, + table_map usable_tables) { if ((left->used_tables() & usable_tables) && !(right->used_tables() & RAND_TABLE_BIT) && @@ -565,7 +627,6 @@ } } - /* Store possible eq field */ (*eq_dep)->type= Module_dep::MODULE_EXPRESSION; //psergey-todo; if (!((*eq_dep)->field= get_field_value(te, field))) return TRUE; @@ -651,7 +712,8 @@ table_map deps_map) { Outer_join_module *oj_dep; - oj_dep= new Outer_join_module(outer_join, my_count_bits(deps_map)); + if (!(oj_dep= new Outer_join_module(outer_join, my_count_bits(deps_map)))) + return NULL; te->n_outer_joins++; /* === modified file 'sql/sql_select.cc' --- a/sql/sql_select.cc 2009-08-13 21:12:12 +0000 +++ b/sql/sql_select.cc 2009-08-16 14:35:47 +0000 @@ -8967,20 +8967,6 @@ JOIN *join= last->join; while (last_emb) { - /* - psergey-elim: (nevermind) - new_prefix= cur_prefix & ~last; - if (!(new_prefix & cur_table_map)) // removed last inner table - { - join->cur_embedding_map&= ~last_emb->nested_join->nj_map; - } - else (current) - { - // Won't hurt doing it all the time: - join->cur_embedding_map |= ...; - } - else - */ if (!(--last_emb->nested_join->counter)) join->cur_embedding_map&= ~last_emb->nested_join->nj_map; else if (last_emb->nested_join->n_tables-1 ==

1 0

[Maria-developers] Updated (by Guest): Add a mysqlbinlog option to change the used database (36)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add a mysqlbinlog option to change the used database CREATION DATE..: Fri, 07 Aug 2009, 14:57 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 36 (http://askmonty.org/worklog/?tid=36) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Sun, 16 Aug 2009, 17:11)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.27162 2009-08-16 17:11:12.000000000 +0300 +++ /tmp/wklog.36.new.27162 2009-08-16 17:11:12.000000000 +0300 @@ -13,6 +13,8 @@ statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). +See also MySQL BUG#42941. + What we could do ---------------- -=-=(Psergey - Mon, 10 Aug 2009, 15:41)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.13035 2009-08-10 15:41:51.000000000 +0300 +++ /tmp/wklog.36.new.13035 2009-08-10 15:41:51.000000000 +0300 @@ -1,5 +1,7 @@ Context ------- +(See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global +overview) At the moment, the server has a replication slave option --replicate-rewrite-db="from->to" -=-=(Guest - Mon, 10 Aug 2009, 11:12)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.6580 2009-08-10 11:12:36.000000000 +0300 +++ /tmp/wklog.36.new.6580 2009-08-10 11:12:36.000000000 +0300 @@ -1,4 +1,3 @@ - Context ------- At the moment, the server has a replication slave option @@ -67,6 +66,6 @@ It will be possible to do the rewrites either on the slave ( --replicate-rewrite-db will work for all kinds of statements), or in -mysqlbinlog (adding a comment is easy and doesn't require use to parse the -statement). +mysqlbinlog (adding a comment is easy and doesn't require mysqlbinlog to +parse the statement). -=-=(Psergey - Sun, 09 Aug 2009, 23:53)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.13425 2009-08-09 23:53:54.000000000 +0300 +++ /tmp/wklog.36.new.13425 2009-08-09 23:53:54.000000000 +0300 @@ -1 +1,72 @@ +Context +------- +At the moment, the server has a replication slave option + + --replicate-rewrite-db="from->to" + +the option affects +- Table_map_log_event (all RBR events) +- Load_log_event (LOAD DATA) +- Query_log_event (SBR-based updates, with the usual assumption that the + statement refers to tables in current database, so that changing the current + database will make the statement to work on a table in a different database). + +What we could do +---------------- + +Option1: make mysqlbinlog accept --replicate-rewrite-db option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Make mysqlbinlog accept --replicate-rewrite-db options and process them to the +same extent as replication slave would process --replicate-rewrite-db option. + + +Option2: Add database-agnostic RBR events and --strip-db option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Right now RBR events require a databasename. It is not possible to have RBR +event stream that won't mention which database the events are for. When I +tried to use debugger and specify empty database name, attempt to apply the +binlog resulted in this error: + +090809 17:38:44 [ERROR] Slave SQL: Error 'Table '.tablename' doesn't exist' on +opening tables, + +We could do as follows: +- Make the server interpret empty database name in RBR event (i.e. in a + Table_map_log_event) as "use current database". Binlog slave thread + probably should not allow such events as it doesn't have a natural current + database. +- Add a mysqlbinlog --strip-db option that would + = not produce any "USE dbname" statements + = change databasename for all RBR events to be empty + +That way, mysqlbinlog output will be database-agnostic and apply to the +current database. +(this will have the usual limitations that we assume that all statements in +the binlog refer to the current database). + +Option3: Enhance database rewrite +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +If there is a need to support database change for statements that use +dbname.tablename notation and are replicated as statements (i.e. are DDL +statements and/or DML statements that are binlogged as statements), +then that could be supported as follows: + +- Make the server's parser recognize special form of comments + + /* !database-alias(oldname,newname) */ + + and save the mapping somewhere + +- Put the hooks in table open and name resolution code to use the saved + mapping. + + +Once we've done the above, it will be easy to perform a complete, +no-compromise or restrictions database name change in binary log. + +It will be possible to do the rewrites either on the slave ( +--replicate-rewrite-db will work for all kinds of statements), or in +mysqlbinlog (adding a comment is easy and doesn't require use to parse the +statement). + -=-=(Psergey - Sun, 09 Aug 2009, 12:27)=-=- Dependency created: 39 now depends on 36 -=-=(Psergey - Fri, 07 Aug 2009, 14:57)=-=- Title modified. --- /tmp/wklog.36.old.14687 2009-08-07 14:57:49.000000000 +0300 +++ /tmp/wklog.36.new.14687 2009-08-07 14:57:49.000000000 +0300 @@ -1 +1 @@ -Add a mysqlbinlog option to change the database +Add a mysqlbinlog option to change the used database DESCRIPTION: Sometimes there is a need to take a binary log and apply it to a database with a different name than the original name of the database on binlog producer. If one is using statement-based replication, he can achieve this by grepping out "USE dbname" statements out of the output of mysqlbinlog(*). With row-based replication this is no longer possible, as database name is encoded within the the BINLOG '....' statement. This task is about adding an option to mysqlbinlog that would allow to change the names of used databases in both RBR and SBR events. (*) this implies that all statements refer to tables in the current database, doesn't catch updates made inside stored functions and so forth, but still works for a practially-important subset of cases. HIGH-LEVEL SPECIFICATION: Context ------- (See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global overview) At the moment, the server has a replication slave option --replicate-rewrite-db="from->to" the option affects - Table_map_log_event (all RBR events) - Load_log_event (LOAD DATA) - Query_log_event (SBR-based updates, with the usual assumption that the statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). See also MySQL BUG#42941. What we could do ---------------- Option1: make mysqlbinlog accept --replicate-rewrite-db option ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Make mysqlbinlog accept --replicate-rewrite-db options and process them to the same extent as replication slave would process --replicate-rewrite-db option. Option2: Add database-agnostic RBR events and --strip-db option ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Right now RBR events require a databasename. It is not possible to have RBR event stream that won't mention which database the events are for. When I tried to use debugger and specify empty database name, attempt to apply the binlog resulted in this error: 090809 17:38:44 [ERROR] Slave SQL: Error 'Table '.tablename' doesn't exist' on opening tables, We could do as follows: - Make the server interpret empty database name in RBR event (i.e. in a Table_map_log_event) as "use current database". Binlog slave thread probably should not allow such events as it doesn't have a natural current database. - Add a mysqlbinlog --strip-db option that would = not produce any "USE dbname" statements = change databasename for all RBR events to be empty That way, mysqlbinlog output will be database-agnostic and apply to the current database. (this will have the usual limitations that we assume that all statements in the binlog refer to the current database). Option3: Enhance database rewrite ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If there is a need to support database change for statements that use dbname.tablename notation and are replicated as statements (i.e. are DDL statements and/or DML statements that are binlogged as statements), then that could be supported as follows: - Make the server's parser recognize special form of comments /* !database-alias(oldname,newname) */ and save the mapping somewhere - Put the hooks in table open and name resolution code to use the saved mapping. Once we've done the above, it will be easy to perform a complete, no-compromise or restrictions database name change in binary log. It will be possible to do the rewrites either on the slave ( --replicate-rewrite-db will work for all kinds of statements), or in mysqlbinlog (adding a comment is easy and doesn't require mysqlbinlog to parse the statement). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Add a mysqlbinlog option to change the used database (36)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add a mysqlbinlog option to change the used database CREATION DATE..: Fri, 07 Aug 2009, 14:57 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 36 (http://askmonty.org/worklog/?tid=36) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Sun, 16 Aug 2009, 17:11)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.27162 2009-08-16 17:11:12.000000000 +0300 +++ /tmp/wklog.36.new.27162 2009-08-16 17:11:12.000000000 +0300 @@ -13,6 +13,8 @@ statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). +See also MySQL BUG#42941. + What we could do ---------------- -=-=(Psergey - Mon, 10 Aug 2009, 15:41)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.13035 2009-08-10 15:41:51.000000000 +0300 +++ /tmp/wklog.36.new.13035 2009-08-10 15:41:51.000000000 +0300 @@ -1,5 +1,7 @@ Context ------- +(See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global +overview) At the moment, the server has a replication slave option --replicate-rewrite-db="from->to" -=-=(Guest - Mon, 10 Aug 2009, 11:12)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.6580 2009-08-10 11:12:36.000000000 +0300 +++ /tmp/wklog.36.new.6580 2009-08-10 11:12:36.000000000 +0300 @@ -1,4 +1,3 @@ - Context ------- At the moment, the server has a replication slave option @@ -67,6 +66,6 @@ It will be possible to do the rewrites either on the slave ( --replicate-rewrite-db will work for all kinds of statements), or in -mysqlbinlog (adding a comment is easy and doesn't require use to parse the -statement). +mysqlbinlog (adding a comment is easy and doesn't require mysqlbinlog to +parse the statement). -=-=(Psergey - Sun, 09 Aug 2009, 23:53)=-=- High-Level Specification modified. --- /tmp/wklog.36.old.13425 2009-08-09 23:53:54.000000000 +0300 +++ /tmp/wklog.36.new.13425 2009-08-09 23:53:54.000000000 +0300 @@ -1 +1,72 @@ +Context +------- +At the moment, the server has a replication slave option + + --replicate-rewrite-db="from->to" + +the option affects +- Table_map_log_event (all RBR events) +- Load_log_event (LOAD DATA) +- Query_log_event (SBR-based updates, with the usual assumption that the + statement refers to tables in current database, so that changing the current + database will make the statement to work on a table in a different database). + +What we could do +---------------- + +Option1: make mysqlbinlog accept --replicate-rewrite-db option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Make mysqlbinlog accept --replicate-rewrite-db options and process them to the +same extent as replication slave would process --replicate-rewrite-db option. + + +Option2: Add database-agnostic RBR events and --strip-db option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Right now RBR events require a databasename. It is not possible to have RBR +event stream that won't mention which database the events are for. When I +tried to use debugger and specify empty database name, attempt to apply the +binlog resulted in this error: + +090809 17:38:44 [ERROR] Slave SQL: Error 'Table '.tablename' doesn't exist' on +opening tables, + +We could do as follows: +- Make the server interpret empty database name in RBR event (i.e. in a + Table_map_log_event) as "use current database". Binlog slave thread + probably should not allow such events as it doesn't have a natural current + database. +- Add a mysqlbinlog --strip-db option that would + = not produce any "USE dbname" statements + = change databasename for all RBR events to be empty + +That way, mysqlbinlog output will be database-agnostic and apply to the +current database. +(this will have the usual limitations that we assume that all statements in +the binlog refer to the current database). + +Option3: Enhance database rewrite +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +If there is a need to support database change for statements that use +dbname.tablename notation and are replicated as statements (i.e. are DDL +statements and/or DML statements that are binlogged as statements), +then that could be supported as follows: + +- Make the server's parser recognize special form of comments + + /* !database-alias(oldname,newname) */ + + and save the mapping somewhere + +- Put the hooks in table open and name resolution code to use the saved + mapping. + + +Once we've done the above, it will be easy to perform a complete, +no-compromise or restrictions database name change in binary log. + +It will be possible to do the rewrites either on the slave ( +--replicate-rewrite-db will work for all kinds of statements), or in +mysqlbinlog (adding a comment is easy and doesn't require use to parse the +statement). + -=-=(Psergey - Sun, 09 Aug 2009, 12:27)=-=- Dependency created: 39 now depends on 36 -=-=(Psergey - Fri, 07 Aug 2009, 14:57)=-=- Title modified. --- /tmp/wklog.36.old.14687 2009-08-07 14:57:49.000000000 +0300 +++ /tmp/wklog.36.new.14687 2009-08-07 14:57:49.000000000 +0300 @@ -1 +1 @@ -Add a mysqlbinlog option to change the database +Add a mysqlbinlog option to change the used database DESCRIPTION: Sometimes there is a need to take a binary log and apply it to a database with a different name than the original name of the database on binlog producer. If one is using statement-based replication, he can achieve this by grepping out "USE dbname" statements out of the output of mysqlbinlog(*). With row-based replication this is no longer possible, as database name is encoded within the the BINLOG '....' statement. This task is about adding an option to mysqlbinlog that would allow to change the names of used databases in both RBR and SBR events. (*) this implies that all statements refer to tables in the current database, doesn't catch updates made inside stored functions and so forth, but still works for a practially-important subset of cases. HIGH-LEVEL SPECIFICATION: Context ------- (See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global overview) At the moment, the server has a replication slave option --replicate-rewrite-db="from->to" the option affects - Table_map_log_event (all RBR events) - Load_log_event (LOAD DATA) - Query_log_event (SBR-based updates, with the usual assumption that the statement refers to tables in current database, so that changing the current database will make the statement to work on a table in a different database). See also MySQL BUG#42941. What we could do ---------------- Option1: make mysqlbinlog accept --replicate-rewrite-db option ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Make mysqlbinlog accept --replicate-rewrite-db options and process them to the same extent as replication slave would process --replicate-rewrite-db option. Option2: Add database-agnostic RBR events and --strip-db option ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Right now RBR events require a databasename. It is not possible to have RBR event stream that won't mention which database the events are for. When I tried to use debugger and specify empty database name, attempt to apply the binlog resulted in this error: 090809 17:38:44 [ERROR] Slave SQL: Error 'Table '.tablename' doesn't exist' on opening tables, We could do as follows: - Make the server interpret empty database name in RBR event (i.e. in a Table_map_log_event) as "use current database". Binlog slave thread probably should not allow such events as it doesn't have a natural current database. - Add a mysqlbinlog --strip-db option that would = not produce any "USE dbname" statements = change databasename for all RBR events to be empty That way, mysqlbinlog output will be database-agnostic and apply to the current database. (this will have the usual limitations that we assume that all statements in the binlog refer to the current database). Option3: Enhance database rewrite ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If there is a need to support database change for statements that use dbname.tablename notation and are replicated as statements (i.e. are DDL statements and/or DML statements that are binlogged as statements), then that could be supported as follows: - Make the server's parser recognize special form of comments /* !database-alias(oldname,newname) */ and save the mapping somewhere - Put the hooks in table open and name resolution code to use the saved mapping. Once we've done the above, it will be easy to perform a complete, no-compromise or restrictions database name change in binary log. It will be possible to do the rewrites either on the slave ( --replicate-rewrite-db will work for all kinds of statements), or in mysqlbinlog (adding a comment is easy and doesn't require mysqlbinlog to parse the statement). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table elimination (17)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: 9.x STATUS.........: In-Progress PRIORITY.......: 60 WORKED HOURS...: 1 ESTIMATE.......: 3 (hours remain) ORIG. ESTIMATE.: 3 PROGRESS NOTES: -=-=(Guest - Sun, 16 Aug 2009, 16:16)=-=- Category updated. --- /tmp/wklog.17.old.24882 2009-08-16 16:16:49.000000000 +0300 +++ /tmp/wklog.17.new.24882 2009-08-16 16:16:49.000000000 +0300 @@ -1 +1 @@ -Client-BackLog +Server-Sprint -=-=(Guest - Sun, 16 Aug 2009, 16:16)=-=- Version updated. --- /tmp/wklog.17.old.24882 2009-08-16 16:16:49.000000000 +0300 +++ /tmp/wklog.17.new.24882 2009-08-16 16:16:49.000000000 +0300 @@ -1 +1 @@ -Server-5.1 +9.x -=-=(Guest - Wed, 29 Jul 2009, 21:41)=-=- Low Level Design modified. --- /tmp/wklog.17.old.26011 2009-07-29 21:41:04.000000000 +0300 +++ /tmp/wklog.17.new.26011 2009-07-29 21:41:04.000000000 +0300 @@ -2,163 +2,146 @@ ~maria-captains/maria/maria-5.1-table-elimination tree. <contents> -1. Conditions for removal -1.1 Quick check if there are candidates -2. Removal operation properties -3. Removal operation -4. User interface -5. Tests and benchmarks -6. Todo, issues to resolve -6.1 To resolve -6.2 Resolved -7. Additional issues +1. Elimination criteria +2. No outside references check +2.1 Quick check if there are tables with no outside references +3. One-match check +3.1 Functional dependency source #1: Potential eq_ref access +3.2 Functional dependency source #2: col2=func(col1) +3.3 Functional dependency source #3: One or zero records in the table +3.4 Functional dependency check implementation +3.4.1 Equality collection: Option1 +3.4.2 Equality collection: Option2 +3.4.3 Functional dependency propagation - option 1 +3.4.4 Functional dependency propagation - option 2 +4. Removal operation properties +5. Removal operation +6. User interface +6.1 @@optimizer_switch flag +6.2 EXPLAIN [EXTENDED] +7. Miscellaneous adjustments +7.1 Fix used_tables() of aggregate functions +7.2 Make subquery predicates collect their outer references +8. Other concerns +8.1 Relationship with outer->inner joins converter +8.2 Relationship with prepared statements +8.3 Relationship with constant table detection +9. Tests and benchmarks </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. -1. Conditions for removal -------------------------- -We can eliminate an inner side of outer join if: -1. For each record combination of outer tables, it will always produce - exactly one record. -2. There are no references to columns of the inner tables anywhere else in +1. Elimination criteria +======================= +We can eliminate inner side of an outer join nest if: + +1. There are no references to columns of the inner tables anywhere else in the query. +2. For each record combination of outer tables, it will always produce + exactly one matching record combination. + +Most of effort in this WL entry is checking these two conditions. -#1 means that every table inside the outer join nest is: - - is a constant table: - = because it can be accessed via eq_ref(const) access, or - = it is a zero-rows or one-row MyISAM-like table [MARK1] - - has an eq_ref access method candidate. - -#2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, - GROUP BY and HAVING do not refer to the inner tables of the outer join - nest. - -1.1 Quick check if there are candidates -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Before we start to enumerate join nests, here is a quick way to check if -there *can be* something to be removed: +2. No outside references check +============================== +Criterion #1 means that the WHERE clause, ON clauses of embedding/subsequent +outer joins, ORDER BY, GROUP BY and HAVING must have no references to inner +tables of the outer join nest we're trying to remove. + +For multi-table UPDATE/DELETE we also must not remove tables that we're +updating/deleting from or tables that are used in UPDATE's SET clause. + +2.1 Quick check if there are tables with no outside references +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Before we start searching for outer join nests that could be eliminated, +we'll do a quick and cheap check if there possibly could be something that +could be eliminated: - if ((tables used in select_list | + if (there are outer joins && + (tables used in select_list | tables used in group/order by UNION | - tables used in where) != bitmap_of_all_tables) + tables used in where) != bitmap_of_all_join_tables) { attempt table elimination; } -2. Removal operation properties -------------------------------- -* There is always one way to remove (no choice to remove either this or that) -* It is always better to remove as much tables as possible (at least within - our cost model). -Thus, no need for any cost calculations/etc. It's an unconditional rewrite. -3. Removal operation --------------------- -* Remove the outer join nest's nested join structure (i.e. get the - outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, - $OJ->embedding->nested_join. Update table_map's of all ancestor nested - joins). [MARK2] +3. One-match check +================== +We can eliminate inner side of outer join if it will always generate exactly +one matching record combination. -* Move the tables and their JOIN_TABs to front like it is done with const - tables, with exception that if eliminated outer join nest was within - another outer join nest, that shouldn't prevent us from moving away the - eliminated tables. +By definition of OUTER JOIN, a NULL-complemented record combination will be +generated when the inner side of outer join has not produced any matches. -* Update join->table_count and all-join-tables bitmap. +What remains to be checked is that there is no possiblity that inner side of +the outer join could produce more than one matching record combination. -* That's it. Nothing else? +We'll refer to one-match property as "functional dependency": -4. User interface ------------------ -* We'll add an @@optimizer switch flag for table elimination. Tentative - name: 'table_elimination'. - (Note ^^ utility of the above questioned ^, as table elimination can never - be worse than no elimination. We're leaning towards not adding the flag) - -* EXPLAIN will not show the removed tables at all. This will allow to check - if tables were removed, and also will behave nicely with anchor model and - VIEWs: stuff that user doesn't care about just won't be there. +- A outer join nest is functionally dependent [wrt outer tables] if it will + produce one matching record combination per each record combination of + outer tables -5. Tests and benchmarks ------------------------ -Create a benchmark in sql-bench which checks if the DBMS has table -elimination. -[According to Monty] Run - - queries that would use elimination - - queries that are very similar to one above (so that they would have same - QEP, execution cost, etc) but cannot use table elimination. -then compare run times and make a conclusion about whether dbms supports table -elimination. +- A table is functionally dependent wrt certain set of dependency tables, if + record combination of dependency tables uniquely identifies zero or one + matching record in the table -6. Todo, issues to resolve --------------------------- +- Definitions of functional dependency of keys (=column tuples) and columns are + apparent. -6.1 To resolve -~~~~~~~~~~~~~~ -- Relationship with prepared statements. - On one hand, it's natural to desire to make table elimination a - once-per-statement operation, like outer->inner join conversion. We'll have - to limit the applicability by removing [MARK1] as that can change during - lifetime of the statement. - - The other option is to do table elimination every time. This will require to - rework operation [MARK2] to be undoable. - - I'm leaning towards doing the former. With anchor modeling, it is unlikely - that we'll meet outer joins which have N inner tables of which some are 1-row - MyISAM tables that do not have primary key. - -6.2 Resolved -~~~~~~~~~~~~ -* outer->inner join conversion is not a problem for table elimination. - We make outer->inner conversions based on predicates in WHERE. If the WHERE - referred to an inner table (requirement for OJ->IJ conversion) then table - elimination would not be applicable anyway. - -* For Multi-table UPDATEs/DELETEs, need to also analyze the SET clause: - - affected tables must not be eliminated - - tables that are used on the right side of the SET x=y assignments must - not be eliminated either. +Our goal is to prove that the entire join nest is functionally-dependent. -* Aggregate functions used to report that they depend on all tables, that is, +Join nest is functionally dependent (on the otside tables) if each of its +elements (those can be either base tables or join nests) is functionally +dependent. - item_agg_func->used_tables() == (1ULL << join->tables) - 1 +Functional dependency is transitive: if table A is f-dependent on the outer +tables and table B is f.dependent on {A, outer_tables} then B is functionally +dependent on the outer tables. + +Subsequent sections list cases when we can declare a table to be +functionally-dependent. + +3.1 Functional dependency source #1: Potential eq_ref access +------------------------------------------------------------ +This is the most practically-important case. Taking the example from the HLD +of this WL entry: + + select + A.colA + from + tableA A + left outer join + tableB B + on + B.id = A.id; - always. Fixed it, now aggregate function reports it depends on - tables that its arguments depend on. In particular, COUNT(*) reports - that it depends on no tables (item_count_star->used_tables()==0). - One consequence of that is that "item->used_tables()==0" is not - equivalent to "item->const_item()==true" anymore (not sure if it's - "anymore" or this has been already happening). - -* EXPLAIN EXTENDED warning text was generated after the JOIN object has - been discarded. This didn't allow to use information about join plan - when printing the warning. Fixed this by keeping the JOIN objects until - we've printed the warning (have also an intent to remove the const - tables from the join output). - -7. Additional issues --------------------- -* We remove ON clauses within outer join nests. If these clauses contain - subqueries, they probably should be gone from EXPLAIN output also? - Yes. Current approach: when removing an outer join nest, walk the ON clause - and mark subselects as eliminated. Then let EXPLAIN code check if the - SELECT was eliminated before the printing (EXPLAIN is generated by doing - a recursive descent, so the check will also cause children of eliminated - selects not to be printed) - -* Table elimination is performed after constant table detection (but before - the range analysis). Constant tables are technically different from - eliminated ones (e.g. the former are shown in EXPLAIN and the latter aren't). - Considering we've already done the join_read_const_table() call, is there any - real difference between constant table and eliminated one? If there is, should - we mark const tables also as eliminated? - from user/EXPLAIN point of view: no. constant table is the one that we read - one record from. eliminated table is the one that we don't acccess at all. +and generalizing it: a table TBL is functionally-dependent if the ON +expression allows to construct a potential eq_ref access to table TBL that +uses only outer or functionally-dependent tables. + +In other words: table TBL will have one match if the ON expression can be +converted into this form + + TBL.unique_key=func(one_match_tables) AND .. remainder ... + +(with appropriate extension for multi-part keys), where + + one_match_tables= { + tables that are not on the inner side of the outer join in question, and + functionally dependent tables + } + +Note that this will cover constant tables, except those that are constant because +they have 0/1 record or are partitioned and have no used partitions. + + +3.2 Functional dependency source #2: col2=func(col1) +---------------------------------------------------- +This comes from the second example in the HLS: -* What is described above will not be able to eliminate this outer join create unique index idx on tableB (id, fromDate); ... left outer join @@ -169,32 +152,331 @@ B.fromDate = (select max(sub.fromDate) from tableB sub where sub.id = A.id); - This is because condition "B.fromDate= func(tableB)" cannot be used. - Reason#1: update_ref_and_keys() does not consider such conditions to - be of any use (and indeed they are not usable for ref access) - so they are not put into KEYUSE array. - Reason#2: even if they were put there, we would need to be able to tell - between predicates like - B.fromDate= func(B.id) // guarantees only one matching row as - // B.id is already bound by B.id=A.id - // hence B.fromDate becomes bound too. - and - "B.fromDate= func(B.*)" // Can potentially have many matching - // records. - We need to - - Have update_ref_and_keys() create KEYUSE elements for such equalities - - Have eliminate_tables() and friends make a more accurate check. - The right check is to check whether all parts of a unique key are bound. - If we have keypartX to be bound, then t.keypartY=func(keypartX) makes - keypartY to be bound. - The difficulty here is that correlated subquery predicate cannot tell what - columns it depends on (it only remembers tables). - Traversing the predicate is expensive and complicated. - We're leaning towards making each subquery predicate have a List<Item> with - items that - - are in the current select - - and it depends on. - This list will be useful in certain other subquery optimizations as well, - it is cheap to collect it in fix_fields() phase, so it will be collected - for every subquery predicate. +Here it is apparent that tableB can be eliminated. It is not possible to +construct eq_ref access to tableB, though, because for the second part of the +primary key (fromDate column) we only got a condition in this form: + + B.fromDate= func(tableB) + +(we write "func(tableB)" because ref optimizer can only determine which tables +the right part of the equality depends on). + +In general case, equality like this doesn't guarantee functional dependency. +For example, if func() == { return fromDate;}, i.e the ON expression is + + ... ON B.id = A.id and B.fromDate = B.fromDate + +then that would allow table B to have multiple matches per record of table A. + +In order to be able to distinguish between these two cases, we'll need to go +down to column level: + +- A table is functionally dependent if it has a unique key that's functionally + dependent + +- A unique key is functionally dependent when all of its columns are + functionally dependent + +- A table column is functionally dependent if the ON clause allows to extract + an AND-part in this form: + + tbl.column = f(functionally-dependent columns or columns of outer tables) + +3.3 Functional dependency source #3: One or zero records in the table +--------------------------------------------------------------------- +A table with one or zero records cannot generate more than one matching +record. This source is of lesser importance as one/zero-record tables are only +MyISAM tables. + +3.4 Functional dependency check implementation +---------------------------------------------- +As shown above, we need something similar to KEYUSE structures, but not +exactly that (we need things that current ref optimizer considers unusable and +don't need things that it considers usable). + +3.4.1 Equality collection: Option1 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +We could +- extend KEYUSE structures to store all kinds of equalities we need +- change update_ref_and_keys() and co. to collect equalities both for ref + access and for table elimination + = [possibly] Improve [eq_]ref access to be able to use equalities in + form keypart2=func(keypart1) +- process the KEYUSE array both by table elimination and by ref access + optimizer. + ++ This requires less effort. +- Code will have to be changed all over sql_select.cc +- update_ref_and_keys() and co. already do several unrelated things. Hooking + up table elimination will make it even worse. + +3.4.2 Equality collection: Option2 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Alternatively, we could process the WHERE clause totally on our own. ++ Table elimination is standalone and easy to detach module. +- Some code duplication with update_ref_and_keys() and co. + +Having got the equalities, we'll to propagate functional dependency property +to unique keys, tables and, ultimately, join nests. + +3.4.3 Functional dependency propagation - option 1 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Borrow the approach used in constant table detection code: + + do + { + converted= FALSE; + for each table T in join nest + { + if (check_if_functionally_dependent(T)) + converted= TRUE; + } + } while (converted == TRUE); + + check_if_functionally_dependent(T) + { + if (T has eq_ref access based on func_dep_tables) + return TRUE; + + Apply the same do-while loop-based approach to available equalities + T.column1=func(other columns) + to spread the set of functionally-dependent columns. The goal is to get + all columns of a certain unique key to be bound. + } + + +3.4.4 Functional dependency propagation - option 2 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Analyze the ON expression(s) and build a list of + + tbl.field = expr(...) + +equalities. tbl here is a table that belongs to a join nest that could +potentially be eliminated. + +besides those, add to the list + - An element for each unique key in the table that needs to be eliminated + - An element for each table that needs to be eliminated + - An element for each join nest that can be eliminated (i.e. has no + references from outside). + +Then, setup "reverse dependencies": each element should have pointers to +elements that are functionally dependent on it: + +- "tbl.field=expr(...)" equality is functionally dependent on all fields that + are used in "expr(...)" (here we take into account only fields that belong + to tables that can potentially be eliminated). +- a unique key is dependent on all of its components +- a table is dependent on all of its unique keys +- a join nest is dependent on all tables that it contains + +These pointers are stored in form of one bitmap, such that: + + "X depends on Y" == test( bitmap[(X's number)*n_objects + (Y's number)] ) + +Each object also stores a number of dependencies it needs to be satisfied +before it itself is satisfied: + +- "tbl.field=expr(...)" needs all its underlying fields (if a field is + referenced many times it is counted only once) + +- a unique key needs all of its key parts + +- a table needs only one of its unique keys + +- a join nest needs all of its tables + +(TODO: so what do we do when we've marked a table as constant? We'll need to +update the "field=expr(....)" elements that use fields of that table. And the +problem is that we won't know how much to decrement from the counters of those +elements. + +Solution#1: switch to table_map() based approach. +Solution#2: introduce separate elements for each involved field. + field will depend on its table, + "field=expr" will depend on fields. +) + +Besides the above, let each element have a pointer to another element, so that +we can have a linked list of elements. + +After the above structures have been created, we start the main algorithm. + +The first step is to create a list of functionally-dependent elements. We walk +across array of dependencies and mark those elements that are already bound +(i.e. their dependencies are satisfied). At the moment those immediately-bound +are only "field=expr" dependencies that don't refer to any columns that are +not bound. + +The second step is the loop + + while (bound_list is not empty) + { + Take the first bound element F off the list. + Use the bitmap to find out what other elements depended on it + for each such element E + { + if (E becomes bound after F is bound) + add E to the list; + } + } + +The last step is to walk through elements that represent the join nests. Those +that are bound can be eliminated. + +4. Removal operation properties +=============================== +* There is always one way to remove (no choice to remove either this or that) +* It is always better to remove as much tables as possible (at least within + our cost model). +Thus, no need for any cost calculations/etc. It's an unconditional rewrite. + + +5. Removal operation +==================== +(This depends a lot on whether we make table elimination a one-off rewrite or +conditional) + +At the moment table elimination is re-done for each join re-execution, hence +the removal operation is designed not to modify any statement's permanent +members. + +* Remove the outer join nest's nested join structure (i.e. get the + outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, + $OJ->embedding->nested_join. Update table_map's of all ancestor nested + joins). [MARK2] + +* Move the tables and their JOIN_TABs to the front of join order, like it is + done with const tables, with exception that if eliminated outer join nest + was within another outer join nest, that shouldn't prevent us from moving + away the eliminated tables. + +* Update join->table_count and all-join-tables bitmap. + ^ TODO: not true anymore ^ + +* That's it. Nothing else? + +6. User interface +================= + +6.1 @@optimizer_switch flag +--------------------------- +Argument againist adding the flag: +* It is always better to perform table elimination than not to do it. + +Arguments for the flag: +* It is always theoretically possible that the new code will cause unintended + slowdowns. +* Having the flag is useful for QA and comparative benchmarking. + +Decision so far: add the flag under #ifdef. Make the flag be present in debug +builds. + +6.2 EXPLAIN [EXTENDED] +---------------------- +There are two possible options: +1. Show eliminated tables, like we do with const tables. +2. Do not show eliminated tables. + +We chose option 2, because: +- the table is not accessed at all (besides locking it) +- it is more natural for anchor model user - when he's querying an anchor- + and attributes view, he doesn't care about the unused attributes. + +EXPLAIN EXTENDED+SHOW WARNINGS won't show the removed table either. + +NOTE: Before this WL, the warning text was generated after all JOIN objects +have been destroyed. This didn't allow to use information about join plan +when printing the warning. We've fixed this by keeping the JOIN objects until +the warning text has been generated. + +Table elimination removes inner sides of outer join, and logically the ON +clause is also removed. If this clause has any subqueries, they will be +also removed from EXPLAIN output. + +An exception to the above is that if we eliminate a derived table, it will +still be shown in EXPLAIN output. This comes from the fact that the FROM +subqueries are evaluated before table elimination is invoked. +TODO: Is the above ok or still remove parts of FROM subqueries? + +7. Miscellaneous adjustments +============================ + +7.1 Fix used_tables() of aggregate functions +-------------------------------------------- +Aggregate functions used to report that they depend on all tables, that is, + + item_agg_func->used_tables() == (1ULL << join->tables) - 1 + +always. Fixed it, now aggregate function reports that it depends on the +tables that its arguments depend on. In particular, COUNT(*) reports that it +depends on no tables (item_count_star->used_tables()==0). One consequence of +that is that "item->used_tables()==0" is not equivalent to +"item->const_item()==true" anymore (not sure if it's "anymore" or this has +been already so for some items). + +7.2 Make subquery predicates collect their outer references +----------------------------------------------------------- +Per-column functional dependency analysis requires us to take a + + tbl.field = func(...) + +equality and tell which columns of which tables are referred from func(...) +expression. For scalar expressions, this is accomplished by Item::walk()-based +traversal. It should be reasonably cheap (the only practical Item that can be +expensive to traverse seems to be a special case of "col IN (const1,const2, +...)". check if we traverse the long list for such items). + +For correlated subqueries, traversal can be expensive, it is cheaper to make +each subquery item have a list of its outer references. The list can be +collected at fix_fields() stage with very little extra cost, and then it could +be used for other optimizations. + + +8. Other concerns +================= + +8.1 Relationship with outer->inner joins converter +-------------------------------------------------- +One could suspect that outer->inner join conversion could get in the way +of table elimination by changing outer joins (which could be eliminated) +to inner (which we will not try to eliminate). +This concern is not valid: we make outer->inner conversions based on +predicates in WHERE. If the WHERE referred to an inner table (this is a +requirement for the conversion) then table elimination would not be +applicable anyway. + +8.2 Relationship with prepared statements +----------------------------------------- +On one hand, it's natural to desire to make table elimination a +once-per-statement operation, like outer->inner join conversion. We'll have +to limit the applicability by removing [MARK1] as that can change during +lifetime of the statement. + +The other option is to do table elimination every time. This will require to +rework operation [MARK2] to be undoable. + + +8.3 Relationship with constant table detection +---------------------------------------------- +Table elimination is performed after constant table detection (but before +the range analysis). Constant tables are technically different from +eliminated ones (e.g. the former are shown in EXPLAIN and the latter aren't). +Considering we've already done the join_read_const_table() call, is there any +real difference between constant table and eliminated one? If there is, should +we mark const tables also as eliminated? +from user/EXPLAIN point of view: no. constant table is the one that we read +one record from. eliminated table is the one that we don't acccess at all. +TODO + +9. Tests and benchmarks +======================= +Create a benchmark in sql-bench which checks if the DBMS has table +elimination. +[According to Monty] Run + - query Q1 that would use elimination + - query Q2 that is very similar to Q1 (so that they would have same + QEP, execution cost, etc) but cannot use table elimination. +then compare run times and make a conclusion about whether the used dbms +supports table elimination. -=-=(Guest - Thu, 23 Jul 2009, 20:07)=-=- Dependency created: 29 now depends on 17 -=-=(Monty - Thu, 23 Jul 2009, 09:19)=-=- Version updated. --- /tmp/wklog.17.old.24090 2009-07-23 09:19:32.000000000 +0300 +++ /tmp/wklog.17.new.24090 2009-07-23 09:19:32.000000000 +0300 @@ -1 +1 @@ -Server-9.x +Server-5.1 -=-=(Guest - Mon, 20 Jul 2009, 14:28)=-=- deukje weg Worked 1 hour and estimate 3 hours remain (original estimate increased by 4 hours). -=-=(Guest - Fri, 17 Jul 2009, 02:44)=-=- Version updated. --- /tmp/wklog.17.old.24138 2009-07-17 02:44:49.000000000 +0300 +++ /tmp/wklog.17.new.24138 2009-07-17 02:44:49.000000000 +0300 @@ -1 +1 @@ -9.x +Server-9.x -=-=(Guest - Fri, 17 Jul 2009, 02:44)=-=- Version updated. --- /tmp/wklog.17.old.24114 2009-07-17 02:44:36.000000000 +0300 +++ /tmp/wklog.17.new.24114 2009-07-17 02:44:36.000000000 +0300 @@ -1 +1 @@ -Server-5.1 +9.x -=-=(Guest - Fri, 17 Jul 2009, 02:44)=-=- Category updated. --- /tmp/wklog.17.old.24114 2009-07-17 02:44:36.000000000 +0300 +++ /tmp/wklog.17.new.24114 2009-07-17 02:44:36.000000000 +0300 @@ -1 +1 @@ -Server-Sprint +Client-BackLog -=-=(Guest - Thu, 18 Jun 2009, 04:15)=-=- Low Level Design modified. --- /tmp/wklog.17.old.29969 2009-06-18 04:15:23.000000000 +0300 +++ /tmp/wklog.17.new.29969 2009-06-18 04:15:23.000000000 +0300 @@ -158,3 +158,43 @@ from user/EXPLAIN point of view: no. constant table is the one that we read one record from. eliminated table is the one that we don't acccess at all. +* What is described above will not be able to eliminate this outer join + create unique index idx on tableB (id, fromDate); + ... + left outer join + tableB B + on + B.id = A.id + and + B.fromDate = (select max(sub.fromDate) + from tableB sub where sub.id = A.id); + + This is because condition "B.fromDate= func(tableB)" cannot be used. + Reason#1: update_ref_and_keys() does not consider such conditions to + be of any use (and indeed they are not usable for ref access) + so they are not put into KEYUSE array. + Reason#2: even if they were put there, we would need to be able to tell + between predicates like + B.fromDate= func(B.id) // guarantees only one matching row as + // B.id is already bound by B.id=A.id + // hence B.fromDate becomes bound too. + and + "B.fromDate= func(B.*)" // Can potentially have many matching + // records. + We need to + - Have update_ref_and_keys() create KEYUSE elements for such equalities + - Have eliminate_tables() and friends make a more accurate check. + The right check is to check whether all parts of a unique key are bound. + If we have keypartX to be bound, then t.keypartY=func(keypartX) makes + keypartY to be bound. + The difficulty here is that correlated subquery predicate cannot tell what + columns it depends on (it only remembers tables). + Traversing the predicate is expensive and complicated. + We're leaning towards making each subquery predicate have a List<Item> with + items that + - are in the current select + - and it depends on. + This list will be useful in certain other subquery optimizations as well, + it is cheap to collect it in fix_fields() phase, so it will be collected + for every subquery predicate. + ------------------------------------------------------------ -=-=(View All Progress Notes, 28 total)=-=- http://askmonty.org/worklog/index.pl?tid=17&nolimit=1 DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: The code (currently in development) is at lp: ~maria-captains/maria/maria-5.1-table-elimination tree. <contents> 1. Elimination criteria 2. No outside references check 2.1 Quick check if there are tables with no outside references 3. One-match check 3.1 Functional dependency source #1: Potential eq_ref access 3.2 Functional dependency source #2: col2=func(col1) 3.3 Functional dependency source #3: One or zero records in the table 3.4 Functional dependency check implementation 3.4.1 Equality collection: Option1 3.4.2 Equality collection: Option2 3.4.3 Functional dependency propagation - option 1 3.4.4 Functional dependency propagation - option 2 4. Removal operation properties 5. Removal operation 6. User interface 6.1 @@optimizer_switch flag 6.2 EXPLAIN [EXTENDED] 7. Miscellaneous adjustments 7.1 Fix used_tables() of aggregate functions 7.2 Make subquery predicates collect their outer references 8. Other concerns 8.1 Relationship with outer->inner joins converter 8.2 Relationship with prepared statements 8.3 Relationship with constant table detection 9. Tests and benchmarks </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Elimination criteria ======================= We can eliminate inner side of an outer join nest if: 1. There are no references to columns of the inner tables anywhere else in the query. 2. For each record combination of outer tables, it will always produce exactly one matching record combination. Most of effort in this WL entry is checking these two conditions. 2. No outside references check ============================== Criterion #1 means that the WHERE clause, ON clauses of embedding/subsequent outer joins, ORDER BY, GROUP BY and HAVING must have no references to inner tables of the outer join nest we're trying to remove. For multi-table UPDATE/DELETE we also must not remove tables that we're updating/deleting from or tables that are used in UPDATE's SET clause. 2.1 Quick check if there are tables with no outside references ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Before we start searching for outer join nests that could be eliminated, we'll do a quick and cheap check if there possibly could be something that could be eliminated: if (there are outer joins && (tables used in select_list | tables used in group/order by UNION | tables used in where) != bitmap_of_all_join_tables) { attempt table elimination; } 3. One-match check ================== We can eliminate inner side of outer join if it will always generate exactly one matching record combination. By definition of OUTER JOIN, a NULL-complemented record combination will be generated when the inner side of outer join has not produced any matches. What remains to be checked is that there is no possiblity that inner side of the outer join could produce more than one matching record combination. We'll refer to one-match property as "functional dependency": - A outer join nest is functionally dependent [wrt outer tables] if it will produce one matching record combination per each record combination of outer tables - A table is functionally dependent wrt certain set of dependency tables, if record combination of dependency tables uniquely identifies zero or one matching record in the table - Definitions of functional dependency of keys (=column tuples) and columns are apparent. Our goal is to prove that the entire join nest is functionally-dependent. Join nest is functionally dependent (on the otside tables) if each of its elements (those can be either base tables or join nests) is functionally dependent. Functional dependency is transitive: if table A is f-dependent on the outer tables and table B is f.dependent on {A, outer_tables} then B is functionally dependent on the outer tables. Subsequent sections list cases when we can declare a table to be functionally-dependent. 3.1 Functional dependency source #1: Potential eq_ref access ------------------------------------------------------------ This is the most practically-important case. Taking the example from the HLD of this WL entry: select A.colA from tableA A left outer join tableB B on B.id = A.id; and generalizing it: a table TBL is functionally-dependent if the ON expression allows to construct a potential eq_ref access to table TBL that uses only outer or functionally-dependent tables. In other words: table TBL will have one match if the ON expression can be converted into this form TBL.unique_key=func(one_match_tables) AND .. remainder ... (with appropriate extension for multi-part keys), where one_match_tables= { tables that are not on the inner side of the outer join in question, and functionally dependent tables } Note that this will cover constant tables, except those that are constant because they have 0/1 record or are partitioned and have no used partitions. 3.2 Functional dependency source #2: col2=func(col1) ---------------------------------------------------- This comes from the second example in the HLS: create unique index idx on tableB (id, fromDate); ... left outer join tableB B on B.id = A.id and B.fromDate = (select max(sub.fromDate) from tableB sub where sub.id = A.id); Here it is apparent that tableB can be eliminated. It is not possible to construct eq_ref access to tableB, though, because for the second part of the primary key (fromDate column) we only got a condition in this form: B.fromDate= func(tableB) (we write "func(tableB)" because ref optimizer can only determine which tables the right part of the equality depends on). In general case, equality like this doesn't guarantee functional dependency. For example, if func() == { return fromDate;}, i.e the ON expression is ... ON B.id = A.id and B.fromDate = B.fromDate then that would allow table B to have multiple matches per record of table A. In order to be able to distinguish between these two cases, we'll need to go down to column level: - A table is functionally dependent if it has a unique key that's functionally dependent - A unique key is functionally dependent when all of its columns are functionally dependent - A table column is functionally dependent if the ON clause allows to extract an AND-part in this form: tbl.column = f(functionally-dependent columns or columns of outer tables) 3.3 Functional dependency source #3: One or zero records in the table --------------------------------------------------------------------- A table with one or zero records cannot generate more than one matching record. This source is of lesser importance as one/zero-record tables are only MyISAM tables. 3.4 Functional dependency check implementation ---------------------------------------------- As shown above, we need something similar to KEYUSE structures, but not exactly that (we need things that current ref optimizer considers unusable and don't need things that it considers usable). 3.4.1 Equality collection: Option1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ We could - extend KEYUSE structures to store all kinds of equalities we need - change update_ref_and_keys() and co. to collect equalities both for ref access and for table elimination = [possibly] Improve [eq_]ref access to be able to use equalities in form keypart2=func(keypart1) - process the KEYUSE array both by table elimination and by ref access optimizer. + This requires less effort. - Code will have to be changed all over sql_select.cc - update_ref_and_keys() and co. already do several unrelated things. Hooking up table elimination will make it even worse. 3.4.2 Equality collection: Option2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Alternatively, we could process the WHERE clause totally on our own. + Table elimination is standalone and easy to detach module. - Some code duplication with update_ref_and_keys() and co. Having got the equalities, we'll to propagate functional dependency property to unique keys, tables and, ultimately, join nests. 3.4.3 Functional dependency propagation - option 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Borrow the approach used in constant table detection code: do { converted= FALSE; for each table T in join nest { if (check_if_functionally_dependent(T)) converted= TRUE; } } while (converted == TRUE); check_if_functionally_dependent(T) { if (T has eq_ref access based on func_dep_tables) return TRUE; Apply the same do-while loop-based approach to available equalities T.column1=func(other columns) to spread the set of functionally-dependent columns. The goal is to get all columns of a certain unique key to be bound. } 3.4.4 Functional dependency propagation - option 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Analyze the ON expression(s) and build a list of tbl.field = expr(...) equalities. tbl here is a table that belongs to a join nest that could potentially be eliminated. besides those, add to the list - An element for each unique key in the table that needs to be eliminated - An element for each table that needs to be eliminated - An element for each join nest that can be eliminated (i.e. has no references from outside). Then, setup "reverse dependencies": each element should have pointers to elements that are functionally dependent on it: - "tbl.field=expr(...)" equality is functionally dependent on all fields that are used in "expr(...)" (here we take into account only fields that belong to tables that can potentially be eliminated). - a unique key is dependent on all of its components - a table is dependent on all of its unique keys - a join nest is dependent on all tables that it contains These pointers are stored in form of one bitmap, such that: "X depends on Y" == test( bitmap[(X's number)*n_objects + (Y's number)] ) Each object also stores a number of dependencies it needs to be satisfied before it itself is satisfied: - "tbl.field=expr(...)" needs all its underlying fields (if a field is referenced many times it is counted only once) - a unique key needs all of its key parts - a table needs only one of its unique keys - a join nest needs all of its tables (TODO: so what do we do when we've marked a table as constant? We'll need to update the "field=expr(....)" elements that use fields of that table. And the problem is that we won't know how much to decrement from the counters of those elements. Solution#1: switch to table_map() based approach. Solution#2: introduce separate elements for each involved field. field will depend on its table, "field=expr" will depend on fields. ) Besides the above, let each element have a pointer to another element, so that we can have a linked list of elements. After the above structures have been created, we start the main algorithm. The first step is to create a list of functionally-dependent elements. We walk across array of dependencies and mark those elements that are already bound (i.e. their dependencies are satisfied). At the moment those immediately-bound are only "field=expr" dependencies that don't refer to any columns that are not bound. The second step is the loop while (bound_list is not empty) { Take the first bound element F off the list. Use the bitmap to find out what other elements depended on it for each such element E { if (E becomes bound after F is bound) add E to the list; } } The last step is to walk through elements that represent the join nests. Those that are bound can be eliminated. 4. Removal operation properties =============================== * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 5. Removal operation ==================== (This depends a lot on whether we make table elimination a one-off rewrite or conditional) At the moment table elimination is re-done for each join re-execution, hence the removal operation is designed not to modify any statement's permanent members. * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to the front of join order, like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. ^ TODO: not true anymore ^ * That's it. Nothing else? 6. User interface ================= 6.1 @@optimizer_switch flag --------------------------- Argument againist adding the flag: * It is always better to perform table elimination than not to do it. Arguments for the flag: * It is always theoretically possible that the new code will cause unintended slowdowns. * Having the flag is useful for QA and comparative benchmarking. Decision so far: add the flag under #ifdef. Make the flag be present in debug builds. 6.2 EXPLAIN [EXTENDED] ---------------------- There are two possible options: 1. Show eliminated tables, like we do with const tables. 2. Do not show eliminated tables. We chose option 2, because: - the table is not accessed at all (besides locking it) - it is more natural for anchor model user - when he's querying an anchor- and attributes view, he doesn't care about the unused attributes. EXPLAIN EXTENDED+SHOW WARNINGS won't show the removed table either. NOTE: Before this WL, the warning text was generated after all JOIN objects have been destroyed. This didn't allow to use information about join plan when printing the warning. We've fixed this by keeping the JOIN objects until the warning text has been generated. Table elimination removes inner sides of outer join, and logically the ON clause is also removed. If this clause has any subqueries, they will be also removed from EXPLAIN output. An exception to the above is that if we eliminate a derived table, it will still be shown in EXPLAIN output. This comes from the fact that the FROM subqueries are evaluated before table elimination is invoked. TODO: Is the above ok or still remove parts of FROM subqueries? 7. Miscellaneous adjustments ============================ 7.1 Fix used_tables() of aggregate functions -------------------------------------------- Aggregate functions used to report that they depend on all tables, that is, item_agg_func->used_tables() == (1ULL << join->tables) - 1 always. Fixed it, now aggregate function reports that it depends on the tables that its arguments depend on. In particular, COUNT(*) reports that it depends on no tables (item_count_star->used_tables()==0). One consequence of that is that "item->used_tables()==0" is not equivalent to "item->const_item()==true" anymore (not sure if it's "anymore" or this has been already so for some items). 7.2 Make subquery predicates collect their outer references ----------------------------------------------------------- Per-column functional dependency analysis requires us to take a tbl.field = func(...) equality and tell which columns of which tables are referred from func(...) expression. For scalar expressions, this is accomplished by Item::walk()-based traversal. It should be reasonably cheap (the only practical Item that can be expensive to traverse seems to be a special case of "col IN (const1,const2, ...)". check if we traverse the long list for such items). For correlated subqueries, traversal can be expensive, it is cheaper to make each subquery item have a list of its outer references. The list can be collected at fix_fields() stage with very little extra cost, and then it could be used for other optimizations. 8. Other concerns ================= 8.1 Relationship with outer->inner joins converter -------------------------------------------------- One could suspect that outer->inner join conversion could get in the way of table elimination by changing outer joins (which could be eliminated) to inner (which we will not try to eliminate). This concern is not valid: we make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (this is a requirement for the conversion) then table elimination would not be applicable anyway. 8.2 Relationship with prepared statements ----------------------------------------- On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. 8.3 Relationship with constant table detection ---------------------------------------------- Table elimination is performed after constant table detection (but before the range analysis). Constant tables are technically different from eliminated ones (e.g. the former are shown in EXPLAIN and the latter aren't). Considering we've already done the join_read_const_table() call, is there any real difference between constant table and eliminated one? If there is, should we mark const tables also as eliminated? from user/EXPLAIN point of view: no. constant table is the one that we read one record from. eliminated table is the one that we don't acccess at all. TODO 9. Tests and benchmarks ======================= Create a benchmark in sql-bench which checks if the DBMS has table elimination. [According to Monty] Run - query Q1 that would use elimination - query Q2 that is very similar to Q1 (so that they would have same QEP, execution cost, etc) but cannot use table elimination. then compare run times and make a conclusion about whether the used dbms supports table elimination. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): Table elimination (17)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Table elimination CREATION DATE..: Sun, 10 May 2009, 19:57 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-Sprint TASK ID........: 17 (http://askmonty.org/worklog/?tid=17) VERSION........: 9.x STATUS.........: In-Progress PRIORITY.......: 60 WORKED HOURS...: 1 ESTIMATE.......: 3 (hours remain) ORIG. ESTIMATE.: 3 PROGRESS NOTES: -=-=(Guest - Sun, 16 Aug 2009, 16:16)=-=- Category updated. --- /tmp/wklog.17.old.24882 2009-08-16 16:16:49.000000000 +0300 +++ /tmp/wklog.17.new.24882 2009-08-16 16:16:49.000000000 +0300 @@ -1 +1 @@ -Client-BackLog +Server-Sprint -=-=(Guest - Sun, 16 Aug 2009, 16:16)=-=- Version updated. --- /tmp/wklog.17.old.24882 2009-08-16 16:16:49.000000000 +0300 +++ /tmp/wklog.17.new.24882 2009-08-16 16:16:49.000000000 +0300 @@ -1 +1 @@ -Server-5.1 +9.x -=-=(Guest - Wed, 29 Jul 2009, 21:41)=-=- Low Level Design modified. --- /tmp/wklog.17.old.26011 2009-07-29 21:41:04.000000000 +0300 +++ /tmp/wklog.17.new.26011 2009-07-29 21:41:04.000000000 +0300 @@ -2,163 +2,146 @@ ~maria-captains/maria/maria-5.1-table-elimination tree. <contents> -1. Conditions for removal -1.1 Quick check if there are candidates -2. Removal operation properties -3. Removal operation -4. User interface -5. Tests and benchmarks -6. Todo, issues to resolve -6.1 To resolve -6.2 Resolved -7. Additional issues +1. Elimination criteria +2. No outside references check +2.1 Quick check if there are tables with no outside references +3. One-match check +3.1 Functional dependency source #1: Potential eq_ref access +3.2 Functional dependency source #2: col2=func(col1) +3.3 Functional dependency source #3: One or zero records in the table +3.4 Functional dependency check implementation +3.4.1 Equality collection: Option1 +3.4.2 Equality collection: Option2 +3.4.3 Functional dependency propagation - option 1 +3.4.4 Functional dependency propagation - option 2 +4. Removal operation properties +5. Removal operation +6. User interface +6.1 @@optimizer_switch flag +6.2 EXPLAIN [EXTENDED] +7. Miscellaneous adjustments +7.1 Fix used_tables() of aggregate functions +7.2 Make subquery predicates collect their outer references +8. Other concerns +8.1 Relationship with outer->inner joins converter +8.2 Relationship with prepared statements +8.3 Relationship with constant table detection +9. Tests and benchmarks </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. -1. Conditions for removal -------------------------- -We can eliminate an inner side of outer join if: -1. For each record combination of outer tables, it will always produce - exactly one record. -2. There are no references to columns of the inner tables anywhere else in +1. Elimination criteria +======================= +We can eliminate inner side of an outer join nest if: + +1. There are no references to columns of the inner tables anywhere else in the query. +2. For each record combination of outer tables, it will always produce + exactly one matching record combination. + +Most of effort in this WL entry is checking these two conditions. -#1 means that every table inside the outer join nest is: - - is a constant table: - = because it can be accessed via eq_ref(const) access, or - = it is a zero-rows or one-row MyISAM-like table [MARK1] - - has an eq_ref access method candidate. - -#2 means that WHERE clause, ON clauses of embedding outer joins, ORDER BY, - GROUP BY and HAVING do not refer to the inner tables of the outer join - nest. - -1.1 Quick check if there are candidates -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Before we start to enumerate join nests, here is a quick way to check if -there *can be* something to be removed: +2. No outside references check +============================== +Criterion #1 means that the WHERE clause, ON clauses of embedding/subsequent +outer joins, ORDER BY, GROUP BY and HAVING must have no references to inner +tables of the outer join nest we're trying to remove. + +For multi-table UPDATE/DELETE we also must not remove tables that we're +updating/deleting from or tables that are used in UPDATE's SET clause. + +2.1 Quick check if there are tables with no outside references +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Before we start searching for outer join nests that could be eliminated, +we'll do a quick and cheap check if there possibly could be something that +could be eliminated: - if ((tables used in select_list | + if (there are outer joins && + (tables used in select_list | tables used in group/order by UNION | - tables used in where) != bitmap_of_all_tables) + tables used in where) != bitmap_of_all_join_tables) { attempt table elimination; } -2. Removal operation properties -------------------------------- -* There is always one way to remove (no choice to remove either this or that) -* It is always better to remove as much tables as possible (at least within - our cost model). -Thus, no need for any cost calculations/etc. It's an unconditional rewrite. -3. Removal operation --------------------- -* Remove the outer join nest's nested join structure (i.e. get the - outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, - $OJ->embedding->nested_join. Update table_map's of all ancestor nested - joins). [MARK2] +3. One-match check +================== +We can eliminate inner side of outer join if it will always generate exactly +one matching record combination. -* Move the tables and their JOIN_TABs to front like it is done with const - tables, with exception that if eliminated outer join nest was within - another outer join nest, that shouldn't prevent us from moving away the - eliminated tables. +By definition of OUTER JOIN, a NULL-complemented record combination will be +generated when the inner side of outer join has not produced any matches. -* Update join->table_count and all-join-tables bitmap. +What remains to be checked is that there is no possiblity that inner side of +the outer join could produce more than one matching record combination. -* That's it. Nothing else? +We'll refer to one-match property as "functional dependency": -4. User interface ------------------ -* We'll add an @@optimizer switch flag for table elimination. Tentative - name: 'table_elimination'. - (Note ^^ utility of the above questioned ^, as table elimination can never - be worse than no elimination. We're leaning towards not adding the flag) - -* EXPLAIN will not show the removed tables at all. This will allow to check - if tables were removed, and also will behave nicely with anchor model and - VIEWs: stuff that user doesn't care about just won't be there. +- A outer join nest is functionally dependent [wrt outer tables] if it will + produce one matching record combination per each record combination of + outer tables -5. Tests and benchmarks ------------------------ -Create a benchmark in sql-bench which checks if the DBMS has table -elimination. -[According to Monty] Run - - queries that would use elimination - - queries that are very similar to one above (so that they would have same - QEP, execution cost, etc) but cannot use table elimination. -then compare run times and make a conclusion about whether dbms supports table -elimination. +- A table is functionally dependent wrt certain set of dependency tables, if + record combination of dependency tables uniquely identifies zero or one + matching record in the table -6. Todo, issues to resolve --------------------------- +- Definitions of functional dependency of keys (=column tuples) and columns are + apparent. -6.1 To resolve -~~~~~~~~~~~~~~ -- Relationship with prepared statements. - On one hand, it's natural to desire to make table elimination a - once-per-statement operation, like outer->inner join conversion. We'll have - to limit the applicability by removing [MARK1] as that can change during - lifetime of the statement. - - The other option is to do table elimination every time. This will require to - rework operation [MARK2] to be undoable. - - I'm leaning towards doing the former. With anchor modeling, it is unlikely - that we'll meet outer joins which have N inner tables of which some are 1-row - MyISAM tables that do not have primary key. - -6.2 Resolved -~~~~~~~~~~~~ -* outer->inner join conversion is not a problem for table elimination. - We make outer->inner conversions based on predicates in WHERE. If the WHERE - referred to an inner table (requirement for OJ->IJ conversion) then table - elimination would not be applicable anyway. - -* For Multi-table UPDATEs/DELETEs, need to also analyze the SET clause: - - affected tables must not be eliminated - - tables that are used on the right side of the SET x=y assignments must - not be eliminated either. +Our goal is to prove that the entire join nest is functionally-dependent. -* Aggregate functions used to report that they depend on all tables, that is, +Join nest is functionally dependent (on the otside tables) if each of its +elements (those can be either base tables or join nests) is functionally +dependent. - item_agg_func->used_tables() == (1ULL << join->tables) - 1 +Functional dependency is transitive: if table A is f-dependent on the outer +tables and table B is f.dependent on {A, outer_tables} then B is functionally +dependent on the outer tables. + +Subsequent sections list cases when we can declare a table to be +functionally-dependent. + +3.1 Functional dependency source #1: Potential eq_ref access +------------------------------------------------------------ +This is the most practically-important case. Taking the example from the HLD +of this WL entry: + + select + A.colA + from + tableA A + left outer join + tableB B + on + B.id = A.id; - always. Fixed it, now aggregate function reports it depends on - tables that its arguments depend on. In particular, COUNT(*) reports - that it depends on no tables (item_count_star->used_tables()==0). - One consequence of that is that "item->used_tables()==0" is not - equivalent to "item->const_item()==true" anymore (not sure if it's - "anymore" or this has been already happening). - -* EXPLAIN EXTENDED warning text was generated after the JOIN object has - been discarded. This didn't allow to use information about join plan - when printing the warning. Fixed this by keeping the JOIN objects until - we've printed the warning (have also an intent to remove the const - tables from the join output). - -7. Additional issues --------------------- -* We remove ON clauses within outer join nests. If these clauses contain - subqueries, they probably should be gone from EXPLAIN output also? - Yes. Current approach: when removing an outer join nest, walk the ON clause - and mark subselects as eliminated. Then let EXPLAIN code check if the - SELECT was eliminated before the printing (EXPLAIN is generated by doing - a recursive descent, so the check will also cause children of eliminated - selects not to be printed) - -* Table elimination is performed after constant table detection (but before - the range analysis). Constant tables are technically different from - eliminated ones (e.g. the former are shown in EXPLAIN and the latter aren't). - Considering we've already done the join_read_const_table() call, is there any - real difference between constant table and eliminated one? If there is, should - we mark const tables also as eliminated? - from user/EXPLAIN point of view: no. constant table is the one that we read - one record from. eliminated table is the one that we don't acccess at all. +and generalizing it: a table TBL is functionally-dependent if the ON +expression allows to construct a potential eq_ref access to table TBL that +uses only outer or functionally-dependent tables. + +In other words: table TBL will have one match if the ON expression can be +converted into this form + + TBL.unique_key=func(one_match_tables) AND .. remainder ... + +(with appropriate extension for multi-part keys), where + + one_match_tables= { + tables that are not on the inner side of the outer join in question, and + functionally dependent tables + } + +Note that this will cover constant tables, except those that are constant because +they have 0/1 record or are partitioned and have no used partitions. + + +3.2 Functional dependency source #2: col2=func(col1) +---------------------------------------------------- +This comes from the second example in the HLS: -* What is described above will not be able to eliminate this outer join create unique index idx on tableB (id, fromDate); ... left outer join @@ -169,32 +152,331 @@ B.fromDate = (select max(sub.fromDate) from tableB sub where sub.id = A.id); - This is because condition "B.fromDate= func(tableB)" cannot be used. - Reason#1: update_ref_and_keys() does not consider such conditions to - be of any use (and indeed they are not usable for ref access) - so they are not put into KEYUSE array. - Reason#2: even if they were put there, we would need to be able to tell - between predicates like - B.fromDate= func(B.id) // guarantees only one matching row as - // B.id is already bound by B.id=A.id - // hence B.fromDate becomes bound too. - and - "B.fromDate= func(B.*)" // Can potentially have many matching - // records. - We need to - - Have update_ref_and_keys() create KEYUSE elements for such equalities - - Have eliminate_tables() and friends make a more accurate check. - The right check is to check whether all parts of a unique key are bound. - If we have keypartX to be bound, then t.keypartY=func(keypartX) makes - keypartY to be bound. - The difficulty here is that correlated subquery predicate cannot tell what - columns it depends on (it only remembers tables). - Traversing the predicate is expensive and complicated. - We're leaning towards making each subquery predicate have a List<Item> with - items that - - are in the current select - - and it depends on. - This list will be useful in certain other subquery optimizations as well, - it is cheap to collect it in fix_fields() phase, so it will be collected - for every subquery predicate. +Here it is apparent that tableB can be eliminated. It is not possible to +construct eq_ref access to tableB, though, because for the second part of the +primary key (fromDate column) we only got a condition in this form: + + B.fromDate= func(tableB) + +(we write "func(tableB)" because ref optimizer can only determine which tables +the right part of the equality depends on). + +In general case, equality like this doesn't guarantee functional dependency. +For example, if func() == { return fromDate;}, i.e the ON expression is + + ... ON B.id = A.id and B.fromDate = B.fromDate + +then that would allow table B to have multiple matches per record of table A. + +In order to be able to distinguish between these two cases, we'll need to go +down to column level: + +- A table is functionally dependent if it has a unique key that's functionally + dependent + +- A unique key is functionally dependent when all of its columns are + functionally dependent + +- A table column is functionally dependent if the ON clause allows to extract + an AND-part in this form: + + tbl.column = f(functionally-dependent columns or columns of outer tables) + +3.3 Functional dependency source #3: One or zero records in the table +--------------------------------------------------------------------- +A table with one or zero records cannot generate more than one matching +record. This source is of lesser importance as one/zero-record tables are only +MyISAM tables. + +3.4 Functional dependency check implementation +---------------------------------------------- +As shown above, we need something similar to KEYUSE structures, but not +exactly that (we need things that current ref optimizer considers unusable and +don't need things that it considers usable). + +3.4.1 Equality collection: Option1 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +We could +- extend KEYUSE structures to store all kinds of equalities we need +- change update_ref_and_keys() and co. to collect equalities both for ref + access and for table elimination + = [possibly] Improve [eq_]ref access to be able to use equalities in + form keypart2=func(keypart1) +- process the KEYUSE array both by table elimination and by ref access + optimizer. + ++ This requires less effort. +- Code will have to be changed all over sql_select.cc +- update_ref_and_keys() and co. already do several unrelated things. Hooking + up table elimination will make it even worse. + +3.4.2 Equality collection: Option2 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Alternatively, we could process the WHERE clause totally on our own. ++ Table elimination is standalone and easy to detach module. +- Some code duplication with update_ref_and_keys() and co. + +Having got the equalities, we'll to propagate functional dependency property +to unique keys, tables and, ultimately, join nests. + +3.4.3 Functional dependency propagation - option 1 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Borrow the approach used in constant table detection code: + + do + { + converted= FALSE; + for each table T in join nest + { + if (check_if_functionally_dependent(T)) + converted= TRUE; + } + } while (converted == TRUE); + + check_if_functionally_dependent(T) + { + if (T has eq_ref access based on func_dep_tables) + return TRUE; + + Apply the same do-while loop-based approach to available equalities + T.column1=func(other columns) + to spread the set of functionally-dependent columns. The goal is to get + all columns of a certain unique key to be bound. + } + + +3.4.4 Functional dependency propagation - option 2 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Analyze the ON expression(s) and build a list of + + tbl.field = expr(...) + +equalities. tbl here is a table that belongs to a join nest that could +potentially be eliminated. + +besides those, add to the list + - An element for each unique key in the table that needs to be eliminated + - An element for each table that needs to be eliminated + - An element for each join nest that can be eliminated (i.e. has no + references from outside). + +Then, setup "reverse dependencies": each element should have pointers to +elements that are functionally dependent on it: + +- "tbl.field=expr(...)" equality is functionally dependent on all fields that + are used in "expr(...)" (here we take into account only fields that belong + to tables that can potentially be eliminated). +- a unique key is dependent on all of its components +- a table is dependent on all of its unique keys +- a join nest is dependent on all tables that it contains + +These pointers are stored in form of one bitmap, such that: + + "X depends on Y" == test( bitmap[(X's number)*n_objects + (Y's number)] ) + +Each object also stores a number of dependencies it needs to be satisfied +before it itself is satisfied: + +- "tbl.field=expr(...)" needs all its underlying fields (if a field is + referenced many times it is counted only once) + +- a unique key needs all of its key parts + +- a table needs only one of its unique keys + +- a join nest needs all of its tables + +(TODO: so what do we do when we've marked a table as constant? We'll need to +update the "field=expr(....)" elements that use fields of that table. And the +problem is that we won't know how much to decrement from the counters of those +elements. + +Solution#1: switch to table_map() based approach. +Solution#2: introduce separate elements for each involved field. + field will depend on its table, + "field=expr" will depend on fields. +) + +Besides the above, let each element have a pointer to another element, so that +we can have a linked list of elements. + +After the above structures have been created, we start the main algorithm. + +The first step is to create a list of functionally-dependent elements. We walk +across array of dependencies and mark those elements that are already bound +(i.e. their dependencies are satisfied). At the moment those immediately-bound +are only "field=expr" dependencies that don't refer to any columns that are +not bound. + +The second step is the loop + + while (bound_list is not empty) + { + Take the first bound element F off the list. + Use the bitmap to find out what other elements depended on it + for each such element E + { + if (E becomes bound after F is bound) + add E to the list; + } + } + +The last step is to walk through elements that represent the join nests. Those +that are bound can be eliminated. + +4. Removal operation properties +=============================== +* There is always one way to remove (no choice to remove either this or that) +* It is always better to remove as much tables as possible (at least within + our cost model). +Thus, no need for any cost calculations/etc. It's an unconditional rewrite. + + +5. Removal operation +==================== +(This depends a lot on whether we make table elimination a one-off rewrite or +conditional) + +At the moment table elimination is re-done for each join re-execution, hence +the removal operation is designed not to modify any statement's permanent +members. + +* Remove the outer join nest's nested join structure (i.e. get the + outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, + $OJ->embedding->nested_join. Update table_map's of all ancestor nested + joins). [MARK2] + +* Move the tables and their JOIN_TABs to the front of join order, like it is + done with const tables, with exception that if eliminated outer join nest + was within another outer join nest, that shouldn't prevent us from moving + away the eliminated tables. + +* Update join->table_count and all-join-tables bitmap. + ^ TODO: not true anymore ^ + +* That's it. Nothing else? + +6. User interface +================= + +6.1 @@optimizer_switch flag +--------------------------- +Argument againist adding the flag: +* It is always better to perform table elimination than not to do it. + +Arguments for the flag: +* It is always theoretically possible that the new code will cause unintended + slowdowns. +* Having the flag is useful for QA and comparative benchmarking. + +Decision so far: add the flag under #ifdef. Make the flag be present in debug +builds. + +6.2 EXPLAIN [EXTENDED] +---------------------- +There are two possible options: +1. Show eliminated tables, like we do with const tables. +2. Do not show eliminated tables. + +We chose option 2, because: +- the table is not accessed at all (besides locking it) +- it is more natural for anchor model user - when he's querying an anchor- + and attributes view, he doesn't care about the unused attributes. + +EXPLAIN EXTENDED+SHOW WARNINGS won't show the removed table either. + +NOTE: Before this WL, the warning text was generated after all JOIN objects +have been destroyed. This didn't allow to use information about join plan +when printing the warning. We've fixed this by keeping the JOIN objects until +the warning text has been generated. + +Table elimination removes inner sides of outer join, and logically the ON +clause is also removed. If this clause has any subqueries, they will be +also removed from EXPLAIN output. + +An exception to the above is that if we eliminate a derived table, it will +still be shown in EXPLAIN output. This comes from the fact that the FROM +subqueries are evaluated before table elimination is invoked. +TODO: Is the above ok or still remove parts of FROM subqueries? + +7. Miscellaneous adjustments +============================ + +7.1 Fix used_tables() of aggregate functions +-------------------------------------------- +Aggregate functions used to report that they depend on all tables, that is, + + item_agg_func->used_tables() == (1ULL << join->tables) - 1 + +always. Fixed it, now aggregate function reports that it depends on the +tables that its arguments depend on. In particular, COUNT(*) reports that it +depends on no tables (item_count_star->used_tables()==0). One consequence of +that is that "item->used_tables()==0" is not equivalent to +"item->const_item()==true" anymore (not sure if it's "anymore" or this has +been already so for some items). + +7.2 Make subquery predicates collect their outer references +----------------------------------------------------------- +Per-column functional dependency analysis requires us to take a + + tbl.field = func(...) + +equality and tell which columns of which tables are referred from func(...) +expression. For scalar expressions, this is accomplished by Item::walk()-based +traversal. It should be reasonably cheap (the only practical Item that can be +expensive to traverse seems to be a special case of "col IN (const1,const2, +...)". check if we traverse the long list for such items). + +For correlated subqueries, traversal can be expensive, it is cheaper to make +each subquery item have a list of its outer references. The list can be +collected at fix_fields() stage with very little extra cost, and then it could +be used for other optimizations. + + +8. Other concerns +================= + +8.1 Relationship with outer->inner joins converter +-------------------------------------------------- +One could suspect that outer->inner join conversion could get in the way +of table elimination by changing outer joins (which could be eliminated) +to inner (which we will not try to eliminate). +This concern is not valid: we make outer->inner conversions based on +predicates in WHERE. If the WHERE referred to an inner table (this is a +requirement for the conversion) then table elimination would not be +applicable anyway. + +8.2 Relationship with prepared statements +----------------------------------------- +On one hand, it's natural to desire to make table elimination a +once-per-statement operation, like outer->inner join conversion. We'll have +to limit the applicability by removing [MARK1] as that can change during +lifetime of the statement. + +The other option is to do table elimination every time. This will require to +rework operation [MARK2] to be undoable. + + +8.3 Relationship with constant table detection +---------------------------------------------- +Table elimination is performed after constant table detection (but before +the range analysis). Constant tables are technically different from +eliminated ones (e.g. the former are shown in EXPLAIN and the latter aren't). +Considering we've already done the join_read_const_table() call, is there any +real difference between constant table and eliminated one? If there is, should +we mark const tables also as eliminated? +from user/EXPLAIN point of view: no. constant table is the one that we read +one record from. eliminated table is the one that we don't acccess at all. +TODO + +9. Tests and benchmarks +======================= +Create a benchmark in sql-bench which checks if the DBMS has table +elimination. +[According to Monty] Run + - query Q1 that would use elimination + - query Q2 that is very similar to Q1 (so that they would have same + QEP, execution cost, etc) but cannot use table elimination. +then compare run times and make a conclusion about whether the used dbms +supports table elimination. -=-=(Guest - Thu, 23 Jul 2009, 20:07)=-=- Dependency created: 29 now depends on 17 -=-=(Monty - Thu, 23 Jul 2009, 09:19)=-=- Version updated. --- /tmp/wklog.17.old.24090 2009-07-23 09:19:32.000000000 +0300 +++ /tmp/wklog.17.new.24090 2009-07-23 09:19:32.000000000 +0300 @@ -1 +1 @@ -Server-9.x +Server-5.1 -=-=(Guest - Mon, 20 Jul 2009, 14:28)=-=- deukje weg Worked 1 hour and estimate 3 hours remain (original estimate increased by 4 hours). -=-=(Guest - Fri, 17 Jul 2009, 02:44)=-=- Version updated. --- /tmp/wklog.17.old.24138 2009-07-17 02:44:49.000000000 +0300 +++ /tmp/wklog.17.new.24138 2009-07-17 02:44:49.000000000 +0300 @@ -1 +1 @@ -9.x +Server-9.x -=-=(Guest - Fri, 17 Jul 2009, 02:44)=-=- Version updated. --- /tmp/wklog.17.old.24114 2009-07-17 02:44:36.000000000 +0300 +++ /tmp/wklog.17.new.24114 2009-07-17 02:44:36.000000000 +0300 @@ -1 +1 @@ -Server-5.1 +9.x -=-=(Guest - Fri, 17 Jul 2009, 02:44)=-=- Category updated. --- /tmp/wklog.17.old.24114 2009-07-17 02:44:36.000000000 +0300 +++ /tmp/wklog.17.new.24114 2009-07-17 02:44:36.000000000 +0300 @@ -1 +1 @@ -Server-Sprint +Client-BackLog -=-=(Guest - Thu, 18 Jun 2009, 04:15)=-=- Low Level Design modified. --- /tmp/wklog.17.old.29969 2009-06-18 04:15:23.000000000 +0300 +++ /tmp/wklog.17.new.29969 2009-06-18 04:15:23.000000000 +0300 @@ -158,3 +158,43 @@ from user/EXPLAIN point of view: no. constant table is the one that we read one record from. eliminated table is the one that we don't acccess at all. +* What is described above will not be able to eliminate this outer join + create unique index idx on tableB (id, fromDate); + ... + left outer join + tableB B + on + B.id = A.id + and + B.fromDate = (select max(sub.fromDate) + from tableB sub where sub.id = A.id); + + This is because condition "B.fromDate= func(tableB)" cannot be used. + Reason#1: update_ref_and_keys() does not consider such conditions to + be of any use (and indeed they are not usable for ref access) + so they are not put into KEYUSE array. + Reason#2: even if they were put there, we would need to be able to tell + between predicates like + B.fromDate= func(B.id) // guarantees only one matching row as + // B.id is already bound by B.id=A.id + // hence B.fromDate becomes bound too. + and + "B.fromDate= func(B.*)" // Can potentially have many matching + // records. + We need to + - Have update_ref_and_keys() create KEYUSE elements for such equalities + - Have eliminate_tables() and friends make a more accurate check. + The right check is to check whether all parts of a unique key are bound. + If we have keypartX to be bound, then t.keypartY=func(keypartX) makes + keypartY to be bound. + The difficulty here is that correlated subquery predicate cannot tell what + columns it depends on (it only remembers tables). + Traversing the predicate is expensive and complicated. + We're leaning towards making each subquery predicate have a List<Item> with + items that + - are in the current select + - and it depends on. + This list will be useful in certain other subquery optimizations as well, + it is cheap to collect it in fix_fields() phase, so it will be collected + for every subquery predicate. + ------------------------------------------------------------ -=-=(View All Progress Notes, 28 total)=-=- http://askmonty.org/worklog/index.pl?tid=17&nolimit=1 DESCRIPTION: Eliminate not needed tables from SELECT queries.. This will speed up some views and automatically generated queries. Example: CREATE TABLE B (id int primary key); select A.colA from tableA A left outer join tableB B on B.id = A.id; In this case we can remove table B and the join from the query. HIGH-LEVEL SPECIFICATION: Here is an extended explanation of table elimination. Table elimination is a feature found in some modern query optimizers, of which Microsoft SQL Server 2005/2008 seems to have the most advanced implementation. Oracle 11g has also been confirmed to use table elimination but not to the same extent. Basically, what table elimination does, is to remove tables from the execution plan when it is unnecessary to include them. This can, of course, only happen if the right circumstances arise. Let us for example look at the following query: select A.colA from tableA A left outer join tableB B on B.id = A.id; When using A as the left table we ensure that the query will return at least as many rows as there are in that table. For rows where the join condition (B.id = A.id) is not met the selected column (A.colA) will still contain it's original value. The not seen B.* row would contain all NULL:s. However, the result set could actually contain more rows than what is found in tableA if there are duplicates of the column B.id in tableB. If A contains a row [1, "val1"] and B the rows [1, "other1a"],[1, "other1b"] then two rows will match in the join condition. The only way to know what the result will look like is to actually touch both tables during execution. Instead, let's say that tableB contains rows that make it possible to place a unique constraint on the column B.id, for example and often the case a primary key. In this situation we know that we will get exactly as many rows as there are in tableA, since joining with tableB cannot introduce any duplicates. If further, as in the example query, we do not select any columns from tableB, touching that table during execution is unnecessary. We can remove the whole join operation from the execution plan. Both SQL Server 2005/2008 and Oracle 11g will deploy table elimination in the case described above. Let us look at a more advanced query, where Oracle fails. select A.colA from tableA A left outer join tableB B on B.id = A.id and B.fromDate = ( select max(sub.fromDate) from tableB sub where sub.id = A.id ); In this example we have added another join condition, which ensures that we only pick the matching row from tableB having the latest fromDate. In this case tableB will contain duplicates of the column B.id, so in order to ensure uniqueness the primary key has to contain the fromDate column as well. In other words the primary key of tableB is (B.id, B.fromDate). Furthermore, since the subselect ensures that we only pick the latest B.fromDate for a given B.id we know that at most one row will match the join condition. We will again have the situation where joining with tableB cannot affect the number of rows in the result set. Since we do not select any columns from tableB, the whole join operation can be eliminated from the execution plan. SQL Server 2005/2008 will deploy table elimination in this situation as well. We have not found a way to make Oracle 11g use it for this type of query. Queries like these arise in two situations. Either when you have denormalized model consisting of a fact table with several related dimension tables, or when you have a highly normalized model where each attribute is stored in its own table. The example with the subselect is common whenever you store historized/versioned data. LOW-LEVEL DESIGN: The code (currently in development) is at lp: ~maria-captains/maria/maria-5.1-table-elimination tree. <contents> 1. Elimination criteria 2. No outside references check 2.1 Quick check if there are tables with no outside references 3. One-match check 3.1 Functional dependency source #1: Potential eq_ref access 3.2 Functional dependency source #2: col2=func(col1) 3.3 Functional dependency source #3: One or zero records in the table 3.4 Functional dependency check implementation 3.4.1 Equality collection: Option1 3.4.2 Equality collection: Option2 3.4.3 Functional dependency propagation - option 1 3.4.4 Functional dependency propagation - option 2 4. Removal operation properties 5. Removal operation 6. User interface 6.1 @@optimizer_switch flag 6.2 EXPLAIN [EXTENDED] 7. Miscellaneous adjustments 7.1 Fix used_tables() of aggregate functions 7.2 Make subquery predicates collect their outer references 8. Other concerns 8.1 Relationship with outer->inner joins converter 8.2 Relationship with prepared statements 8.3 Relationship with constant table detection 9. Tests and benchmarks </contents> It's not really about elimination of tables, it's about elimination of inner sides of outer joins. 1. Elimination criteria ======================= We can eliminate inner side of an outer join nest if: 1. There are no references to columns of the inner tables anywhere else in the query. 2. For each record combination of outer tables, it will always produce exactly one matching record combination. Most of effort in this WL entry is checking these two conditions. 2. No outside references check ============================== Criterion #1 means that the WHERE clause, ON clauses of embedding/subsequent outer joins, ORDER BY, GROUP BY and HAVING must have no references to inner tables of the outer join nest we're trying to remove. For multi-table UPDATE/DELETE we also must not remove tables that we're updating/deleting from or tables that are used in UPDATE's SET clause. 2.1 Quick check if there are tables with no outside references ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Before we start searching for outer join nests that could be eliminated, we'll do a quick and cheap check if there possibly could be something that could be eliminated: if (there are outer joins && (tables used in select_list | tables used in group/order by UNION | tables used in where) != bitmap_of_all_join_tables) { attempt table elimination; } 3. One-match check ================== We can eliminate inner side of outer join if it will always generate exactly one matching record combination. By definition of OUTER JOIN, a NULL-complemented record combination will be generated when the inner side of outer join has not produced any matches. What remains to be checked is that there is no possiblity that inner side of the outer join could produce more than one matching record combination. We'll refer to one-match property as "functional dependency": - A outer join nest is functionally dependent [wrt outer tables] if it will produce one matching record combination per each record combination of outer tables - A table is functionally dependent wrt certain set of dependency tables, if record combination of dependency tables uniquely identifies zero or one matching record in the table - Definitions of functional dependency of keys (=column tuples) and columns are apparent. Our goal is to prove that the entire join nest is functionally-dependent. Join nest is functionally dependent (on the otside tables) if each of its elements (those can be either base tables or join nests) is functionally dependent. Functional dependency is transitive: if table A is f-dependent on the outer tables and table B is f.dependent on {A, outer_tables} then B is functionally dependent on the outer tables. Subsequent sections list cases when we can declare a table to be functionally-dependent. 3.1 Functional dependency source #1: Potential eq_ref access ------------------------------------------------------------ This is the most practically-important case. Taking the example from the HLD of this WL entry: select A.colA from tableA A left outer join tableB B on B.id = A.id; and generalizing it: a table TBL is functionally-dependent if the ON expression allows to construct a potential eq_ref access to table TBL that uses only outer or functionally-dependent tables. In other words: table TBL will have one match if the ON expression can be converted into this form TBL.unique_key=func(one_match_tables) AND .. remainder ... (with appropriate extension for multi-part keys), where one_match_tables= { tables that are not on the inner side of the outer join in question, and functionally dependent tables } Note that this will cover constant tables, except those that are constant because they have 0/1 record or are partitioned and have no used partitions. 3.2 Functional dependency source #2: col2=func(col1) ---------------------------------------------------- This comes from the second example in the HLS: create unique index idx on tableB (id, fromDate); ... left outer join tableB B on B.id = A.id and B.fromDate = (select max(sub.fromDate) from tableB sub where sub.id = A.id); Here it is apparent that tableB can be eliminated. It is not possible to construct eq_ref access to tableB, though, because for the second part of the primary key (fromDate column) we only got a condition in this form: B.fromDate= func(tableB) (we write "func(tableB)" because ref optimizer can only determine which tables the right part of the equality depends on). In general case, equality like this doesn't guarantee functional dependency. For example, if func() == { return fromDate;}, i.e the ON expression is ... ON B.id = A.id and B.fromDate = B.fromDate then that would allow table B to have multiple matches per record of table A. In order to be able to distinguish between these two cases, we'll need to go down to column level: - A table is functionally dependent if it has a unique key that's functionally dependent - A unique key is functionally dependent when all of its columns are functionally dependent - A table column is functionally dependent if the ON clause allows to extract an AND-part in this form: tbl.column = f(functionally-dependent columns or columns of outer tables) 3.3 Functional dependency source #3: One or zero records in the table --------------------------------------------------------------------- A table with one or zero records cannot generate more than one matching record. This source is of lesser importance as one/zero-record tables are only MyISAM tables. 3.4 Functional dependency check implementation ---------------------------------------------- As shown above, we need something similar to KEYUSE structures, but not exactly that (we need things that current ref optimizer considers unusable and don't need things that it considers usable). 3.4.1 Equality collection: Option1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ We could - extend KEYUSE structures to store all kinds of equalities we need - change update_ref_and_keys() and co. to collect equalities both for ref access and for table elimination = [possibly] Improve [eq_]ref access to be able to use equalities in form keypart2=func(keypart1) - process the KEYUSE array both by table elimination and by ref access optimizer. + This requires less effort. - Code will have to be changed all over sql_select.cc - update_ref_and_keys() and co. already do several unrelated things. Hooking up table elimination will make it even worse. 3.4.2 Equality collection: Option2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Alternatively, we could process the WHERE clause totally on our own. + Table elimination is standalone and easy to detach module. - Some code duplication with update_ref_and_keys() and co. Having got the equalities, we'll to propagate functional dependency property to unique keys, tables and, ultimately, join nests. 3.4.3 Functional dependency propagation - option 1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Borrow the approach used in constant table detection code: do { converted= FALSE; for each table T in join nest { if (check_if_functionally_dependent(T)) converted= TRUE; } } while (converted == TRUE); check_if_functionally_dependent(T) { if (T has eq_ref access based on func_dep_tables) return TRUE; Apply the same do-while loop-based approach to available equalities T.column1=func(other columns) to spread the set of functionally-dependent columns. The goal is to get all columns of a certain unique key to be bound. } 3.4.4 Functional dependency propagation - option 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Analyze the ON expression(s) and build a list of tbl.field = expr(...) equalities. tbl here is a table that belongs to a join nest that could potentially be eliminated. besides those, add to the list - An element for each unique key in the table that needs to be eliminated - An element for each table that needs to be eliminated - An element for each join nest that can be eliminated (i.e. has no references from outside). Then, setup "reverse dependencies": each element should have pointers to elements that are functionally dependent on it: - "tbl.field=expr(...)" equality is functionally dependent on all fields that are used in "expr(...)" (here we take into account only fields that belong to tables that can potentially be eliminated). - a unique key is dependent on all of its components - a table is dependent on all of its unique keys - a join nest is dependent on all tables that it contains These pointers are stored in form of one bitmap, such that: "X depends on Y" == test( bitmap[(X's number)*n_objects + (Y's number)] ) Each object also stores a number of dependencies it needs to be satisfied before it itself is satisfied: - "tbl.field=expr(...)" needs all its underlying fields (if a field is referenced many times it is counted only once) - a unique key needs all of its key parts - a table needs only one of its unique keys - a join nest needs all of its tables (TODO: so what do we do when we've marked a table as constant? We'll need to update the "field=expr(....)" elements that use fields of that table. And the problem is that we won't know how much to decrement from the counters of those elements. Solution#1: switch to table_map() based approach. Solution#2: introduce separate elements for each involved field. field will depend on its table, "field=expr" will depend on fields. ) Besides the above, let each element have a pointer to another element, so that we can have a linked list of elements. After the above structures have been created, we start the main algorithm. The first step is to create a list of functionally-dependent elements. We walk across array of dependencies and mark those elements that are already bound (i.e. their dependencies are satisfied). At the moment those immediately-bound are only "field=expr" dependencies that don't refer to any columns that are not bound. The second step is the loop while (bound_list is not empty) { Take the first bound element F off the list. Use the bitmap to find out what other elements depended on it for each such element E { if (E becomes bound after F is bound) add E to the list; } } The last step is to walk through elements that represent the join nests. Those that are bound can be eliminated. 4. Removal operation properties =============================== * There is always one way to remove (no choice to remove either this or that) * It is always better to remove as much tables as possible (at least within our cost model). Thus, no need for any cost calculations/etc. It's an unconditional rewrite. 5. Removal operation ==================== (This depends a lot on whether we make table elimination a one-off rewrite or conditional) At the moment table elimination is re-done for each join re-execution, hence the removal operation is designed not to modify any statement's permanent members. * Remove the outer join nest's nested join structure (i.e. get the outer join's TABLE_LIST object $OJ and remove it from $OJ->embedding, $OJ->embedding->nested_join. Update table_map's of all ancestor nested joins). [MARK2] * Move the tables and their JOIN_TABs to the front of join order, like it is done with const tables, with exception that if eliminated outer join nest was within another outer join nest, that shouldn't prevent us from moving away the eliminated tables. * Update join->table_count and all-join-tables bitmap. ^ TODO: not true anymore ^ * That's it. Nothing else? 6. User interface ================= 6.1 @@optimizer_switch flag --------------------------- Argument againist adding the flag: * It is always better to perform table elimination than not to do it. Arguments for the flag: * It is always theoretically possible that the new code will cause unintended slowdowns. * Having the flag is useful for QA and comparative benchmarking. Decision so far: add the flag under #ifdef. Make the flag be present in debug builds. 6.2 EXPLAIN [EXTENDED] ---------------------- There are two possible options: 1. Show eliminated tables, like we do with const tables. 2. Do not show eliminated tables. We chose option 2, because: - the table is not accessed at all (besides locking it) - it is more natural for anchor model user - when he's querying an anchor- and attributes view, he doesn't care about the unused attributes. EXPLAIN EXTENDED+SHOW WARNINGS won't show the removed table either. NOTE: Before this WL, the warning text was generated after all JOIN objects have been destroyed. This didn't allow to use information about join plan when printing the warning. We've fixed this by keeping the JOIN objects until the warning text has been generated. Table elimination removes inner sides of outer join, and logically the ON clause is also removed. If this clause has any subqueries, they will be also removed from EXPLAIN output. An exception to the above is that if we eliminate a derived table, it will still be shown in EXPLAIN output. This comes from the fact that the FROM subqueries are evaluated before table elimination is invoked. TODO: Is the above ok or still remove parts of FROM subqueries? 7. Miscellaneous adjustments ============================ 7.1 Fix used_tables() of aggregate functions -------------------------------------------- Aggregate functions used to report that they depend on all tables, that is, item_agg_func->used_tables() == (1ULL << join->tables) - 1 always. Fixed it, now aggregate function reports that it depends on the tables that its arguments depend on. In particular, COUNT(*) reports that it depends on no tables (item_count_star->used_tables()==0). One consequence of that is that "item->used_tables()==0" is not equivalent to "item->const_item()==true" anymore (not sure if it's "anymore" or this has been already so for some items). 7.2 Make subquery predicates collect their outer references ----------------------------------------------------------- Per-column functional dependency analysis requires us to take a tbl.field = func(...) equality and tell which columns of which tables are referred from func(...) expression. For scalar expressions, this is accomplished by Item::walk()-based traversal. It should be reasonably cheap (the only practical Item that can be expensive to traverse seems to be a special case of "col IN (const1,const2, ...)". check if we traverse the long list for such items). For correlated subqueries, traversal can be expensive, it is cheaper to make each subquery item have a list of its outer references. The list can be collected at fix_fields() stage with very little extra cost, and then it could be used for other optimizations. 8. Other concerns ================= 8.1 Relationship with outer->inner joins converter -------------------------------------------------- One could suspect that outer->inner join conversion could get in the way of table elimination by changing outer joins (which could be eliminated) to inner (which we will not try to eliminate). This concern is not valid: we make outer->inner conversions based on predicates in WHERE. If the WHERE referred to an inner table (this is a requirement for the conversion) then table elimination would not be applicable anyway. 8.2 Relationship with prepared statements ----------------------------------------- On one hand, it's natural to desire to make table elimination a once-per-statement operation, like outer->inner join conversion. We'll have to limit the applicability by removing [MARK1] as that can change during lifetime of the statement. The other option is to do table elimination every time. This will require to rework operation [MARK2] to be undoable. 8.3 Relationship with constant table detection ---------------------------------------------- Table elimination is performed after constant table detection (but before the range analysis). Constant tables are technically different from eliminated ones (e.g. the former are shown in EXPLAIN and the latter aren't). Considering we've already done the join_read_const_table() call, is there any real difference between constant table and eliminated one? If there is, should we mark const tables also as eliminated? from user/EXPLAIN point of view: no. constant table is the one that we read one record from. eliminated table is the one that we don't acccess at all. TODO 9. Tests and benchmarks ======================= Create a benchmark in sql-bench which checks if the DBMS has table elimination. [According to Monty] Run - query Q1 that would use elimination - query Q2 that is very similar to Q1 (so that they would have same QEP, execution cost, etc) but cannot use table elimination. then compare run times and make a conclusion about whether the used dbms supports table elimination. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Rev 2728: - Better comments in file:///home/psergey/dev/maria-5.1-table-elim-r10/
by Sergey Petrunya 16 Aug '09

16 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r10/ ------------------------------------------------------------ revno: 2728 revision-id: psergey(a)askmonty.org-20090816124331-gd53m2alc0jb3ws4 parent: psergey(a)askmonty.org-20090816121708-v42h3mehvoy4c7yu committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Sun 2009-08-16 15:43:31 +0300 message: - Better comments - Add OOM error checking === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-16 12:17:08 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-16 12:43:31 +0000 @@ -184,18 +184,11 @@ unknown_args= n_children; } /* - Outer join we're representing. This can be a join nest or a one table that + Outer join we're representing. This can be a join nest or one table that is outer join'ed. */ TABLE_LIST *table_list; - /* - Tables within this outer join (and its descendants) that are not yet known - to be functionally dependent. - */ - table_map missing_tables; //psergey-todo: remove - /* All tables within this outer join and its descendants */ - table_map all_tables; //psergey-todo: remove /* Parent eliminable outer join, if any */ Outer_join_module *parent; }; @@ -229,11 +222,11 @@ }; static -void build_eq_deps_for_cond(Table_elimination *te, Equality_module **fdeps, +bool build_eq_deps_for_cond(Table_elimination *te, Equality_module **fdeps, uint *and_level, Item *cond, table_map usable_tables); static -void add_eq_dep(Table_elimination *te, Equality_module **eq_dep, +bool add_eq_dep(Table_elimination *te, Equality_module **eq_dep, uint and_level, Item_func *cond, Item *left, Item *right, table_map usable_tables); @@ -270,7 +263,7 @@ */ static -void build_eq_deps_for_cond(Table_elimination *te, Equality_module **fdeps, +bool build_eq_deps_for_cond(Table_elimination *te, Equality_module **fdeps, uint *and_level, Item *cond, table_map usable_tables) { @@ -285,7 +278,8 @@ Item *item; while ((item=li++)) { - build_eq_deps_for_cond(te, fdeps, and_level, item, usable_tables); + if (build_eq_deps_for_cond(te, fdeps, and_level, item, usable_tables)) + return TRUE; } /* TODO: inject here a "if we have {t.col=const AND t.col=smth_else}, then @@ -297,47 +291,52 @@ else { (*and_level)++; - build_eq_deps_for_cond(te, fdeps, and_level, li++, usable_tables); + if (build_eq_deps_for_cond(te, fdeps, and_level, li++, usable_tables)) + return TRUE; Item *item; while ((item=li++)) { Equality_module *start_key_fields= *fdeps; (*and_level)++; - build_eq_deps_for_cond(te, fdeps, and_level, item, usable_tables); + if (build_eq_deps_for_cond(te, fdeps, and_level, item, usable_tables)) + return TRUE; *fdeps= merge_func_deps(org_key_fields, start_key_fields, *fdeps, ++(*and_level)); } } - return; + return FALSE; } if (cond->type() != Item::FUNC_ITEM) - return; + return FALSE; Item_func *cond_func= (Item_func*) cond; Item **args= cond_func->arguments(); - Item *fld; switch (cond_func->functype()) { case Item_func::IN_FUNC: { if (cond_func->argument_count() == 2) { - add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1], - usable_tables); - add_eq_dep(te, fdeps, *and_level, cond_func, args[1], args[0], - usable_tables); + if (add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1], + usable_tables) || + add_eq_dep(te, fdeps, *and_level, cond_func, args[1], args[0], + usable_tables)) + return TRUE; } } case Item_func::BETWEEN: { + Item *fld; if (!((Item_func_between*)cond)->negated && + (fld= args[0]->real_item())->type() == Item::FIELD_ITEM && args[1]->eq(args[2], ((Item_field*)fld)->field->binary())) { - add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1], - usable_tables); - add_eq_dep(te, fdeps, *and_level, cond_func, args[1], args[0], - usable_tables); + if (add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1], + usable_tables) || + add_eq_dep(te, fdeps, *and_level, cond_func, args[1], args[0], + usable_tables)) + return TRUE; } break; } @@ -353,10 +352,9 @@ case Item_func::ISNULL_FUNC: { Item *tmp=new Item_null; - if (unlikely(!tmp)) // Should never be true - return; - add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1], - usable_tables); + if (!tmp || add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1], + usable_tables)) + return TRUE; break; } case Item_func::MULT_EQUAL_FUNC: @@ -374,8 +372,9 @@ */ while ((item= it++)) { - add_eq_dep(te, fdeps, *and_level, cond_func, item, const_item, - usable_tables); + if (add_eq_dep(te, fdeps, *and_level, cond_func, item, const_item, + usable_tables)) + return TRUE; } } else @@ -395,8 +394,9 @@ { if (!field->eq(item2->field)) { - add_eq_dep(te, fdeps, *and_level, cond_func, item, item2, - usable_tables); + if (add_eq_dep(te, fdeps, *and_level, cond_func, item, item2, + usable_tables)) + return TRUE; } } it.rewind(); @@ -407,6 +407,7 @@ default: break; } + return FALSE; } @@ -536,7 +537,7 @@ */ static -void add_eq_dep(Table_elimination *te, Equality_module **eq_dep, +bool add_eq_dep(Table_elimination *te, Equality_module **eq_dep, uint and_level, Item_func *cond, Item *left, Item *right, table_map usable_tables) { @@ -550,7 +551,7 @@ if (right->result_type() != STRING_RESULT) { if (field->cmp_type() != right->result_type()) - return; + return FALSE; } else { @@ -560,17 +561,19 @@ */ if (field->cmp_type() == STRING_RESULT && ((Field_str*)field)->charset() != cond->compare_collation()) - return; + return FALSE; } } /* Store possible eq field */ (*eq_dep)->type= Module_dep::MODULE_EXPRESSION; //psergey-todo; - (*eq_dep)->field= get_field_value(te, field); + if (!((*eq_dep)->field= get_field_value(te, field))) + return TRUE; (*eq_dep)->expression= right; (*eq_dep)->level= and_level; (*eq_dep)++; } + return FALSE; }

1 0

[Maria-developers] Rev 2727: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r10/
by Sergey Petrunya 16 Aug '09

16 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r10/ ------------------------------------------------------------ revno: 2727 revision-id: psergey(a)askmonty.org-20090816121708-v42h3mehvoy4c7yu parent: psergey(a)askmonty.org-20090816091549-da84w3nlmx8prmvm committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Sun 2009-08-16 15:17:08 +0300 message: MWL#17: Table elimination - Address review feedback: change expression analyzer used to be a copy-paste of ref analyzer. ref analyzer, besides doing ref analysis, also collected info about keys which had sargable predicates. We didn't need that part here. === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-16 09:15:49 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-16 12:17:08 +0000 @@ -233,11 +233,10 @@ uint *and_level, Item *cond, table_map usable_tables); static -void add_eq_dep(Table_elimination *te, - Equality_module **eq_dep, uint and_level, - Item_func *cond, Field *field, - bool eq_func, Item **value, - uint num_values, table_map usable_tables); +void add_eq_dep(Table_elimination *te, Equality_module **eq_dep, + uint and_level, + Item_func *cond, Item *left, Item *right, + table_map usable_tables); static Equality_module *merge_func_deps(Equality_module *start, Equality_module *new_fields, Equality_module *end, uint and_level); @@ -314,87 +313,54 @@ if (cond->type() != Item::FUNC_ITEM) return; + Item_func *cond_func= (Item_func*) cond; - switch (cond_func->select_optimize()) { - case Item_func::OPTIMIZE_NONE: - break; - case Item_func::OPTIMIZE_KEY: - { - Item **values; - // BETWEEN, IN, NE - if (cond_func->key_item()->real_item()->type() == Item::FIELD_ITEM && - !(cond_func->used_tables() & OUTER_REF_TABLE_BIT)) - { - values= cond_func->arguments()+1; - if (cond_func->functype() == Item_func::NE_FUNC && - cond_func->arguments()[1]->real_item()->type() == Item::FIELD_ITEM && - !(cond_func->arguments()[0]->used_tables() & OUTER_REF_TABLE_BIT)) - values--; - DBUG_ASSERT(cond_func->functype() != Item_func::IN_FUNC || - cond_func->argument_count() != 2); - add_eq_dep(te, fdeps, *and_level, cond_func, - ((Item_field*)(cond_func->key_item()->real_item()))->field, - 0, values, - cond_func->argument_count()-1, - usable_tables); - } - if (cond_func->functype() == Item_func::BETWEEN) - { - values= cond_func->arguments(); - for (uint i= 1 ; i < cond_func->argument_count() ; i++) - { - Item_field *field_item; - if (cond_func->arguments()[i]->real_item()->type() == Item::FIELD_ITEM - && - !(cond_func->arguments()[i]->used_tables() & OUTER_REF_TABLE_BIT)) - { - field_item= (Item_field *) (cond_func->arguments()[i]->real_item()); - add_eq_dep(te, fdeps, *and_level, cond_func, - field_item->field, 0, values, 1, usable_tables); - } - } - } - break; - } - case Item_func::OPTIMIZE_OP: - { - bool equal_func=(cond_func->functype() == Item_func::EQ_FUNC || - cond_func->functype() == Item_func::EQUAL_FUNC); + Item **args= cond_func->arguments(); + Item *fld; - if (cond_func->arguments()[0]->real_item()->type() == Item::FIELD_ITEM && - !(cond_func->arguments()[0]->used_tables() & OUTER_REF_TABLE_BIT)) - { - add_eq_dep(te, fdeps, *and_level, cond_func, - ((Item_field*)(cond_func->arguments()[0])->real_item())->field, - equal_func, - cond_func->arguments()+1, 1, usable_tables); - } - if (cond_func->arguments()[1]->real_item()->type() == Item::FIELD_ITEM && - cond_func->functype() != Item_func::LIKE_FUNC && - !(cond_func->arguments()[1]->used_tables() & OUTER_REF_TABLE_BIT)) - { - add_eq_dep(te, fdeps, *and_level, cond_func, - ((Item_field*)(cond_func->arguments()[1])->real_item())->field, - equal_func, - cond_func->arguments(),1,usable_tables); - } - break; - } - case Item_func::OPTIMIZE_NULL: - /* column_name IS [NOT] NULL */ - if (cond_func->arguments()[0]->real_item()->type() == Item::FIELD_ITEM && - !(cond_func->used_tables() & OUTER_REF_TABLE_BIT)) - { - Item *tmp=new Item_null; - if (unlikely(!tmp)) // Should never be true - return; - add_eq_dep(te, fdeps, *and_level, cond_func, - ((Item_field*)(cond_func->arguments()[0])->real_item())->field, - cond_func->functype() == Item_func::ISNULL_FUNC, - &tmp, 1, usable_tables); - } - break; - case Item_func::OPTIMIZE_EQUAL: + switch (cond_func->functype()) { + case Item_func::IN_FUNC: + { + if (cond_func->argument_count() == 2) + { + add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1], + usable_tables); + add_eq_dep(te, fdeps, *and_level, cond_func, args[1], args[0], + usable_tables); + } + } + case Item_func::BETWEEN: + { + if (!((Item_func_between*)cond)->negated && + args[1]->eq(args[2], ((Item_field*)fld)->field->binary())) + { + add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1], + usable_tables); + add_eq_dep(te, fdeps, *and_level, cond_func, args[1], args[0], + usable_tables); + } + break; + } + case Item_func::EQ_FUNC: + case Item_func::EQUAL_FUNC: + { + add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1], + usable_tables); + add_eq_dep(te, fdeps, *and_level, cond_func, args[1], args[0], + usable_tables); + break; + } + case Item_func::ISNULL_FUNC: + { + Item *tmp=new Item_null; + if (unlikely(!tmp)) // Should never be true + return; + add_eq_dep(te, fdeps, *and_level, cond_func, args[0], args[1], + usable_tables); + break; + } + case Item_func::MULT_EQUAL_FUNC: + { Item_equal *item_equal= (Item_equal *) cond; Item *const_item= item_equal->get_const(); Item_equal_iterator it(*item_equal); @@ -408,8 +374,8 @@ */ while ((item= it++)) { - add_eq_dep(te, fdeps, *and_level, cond_func, item->field, - TRUE, &const_item, 1, usable_tables); + add_eq_dep(te, fdeps, *and_level, cond_func, item, const_item, + usable_tables); } } else @@ -424,12 +390,13 @@ while ((item= fi++)) { Field *field= item->field; - while ((item= it++)) + Item_field *item2; + while ((item2= it++)) { - if (!field->eq(item->field)) + if (!field->eq(item2->field)) { - add_eq_dep(te, fdeps, *and_level, cond_func, field, - TRUE, (Item **) &item, 1, usable_tables); + add_eq_dep(te, fdeps, *and_level, cond_func, item, item2, + usable_tables); } } it.rewind(); @@ -437,6 +404,9 @@ } break; } + default: + break; + } } @@ -567,75 +537,40 @@ static void add_eq_dep(Table_elimination *te, Equality_module **eq_dep, - uint and_level, Item_func *cond, Field *field, - bool eq_func, Item **value, uint num_values, - table_map usable_tables) + uint and_level, Item_func *cond, + Item *left, Item *right, table_map usable_tables) { - if (!(field->table->map & usable_tables)) - return; - - for (uint i=0; i<num_values; i++) - { - if ((value[i])->used_tables() & RAND_TABLE_BIT) - return; - } - - /* - Save the following cases: - Field op constant - Field LIKE constant where constant doesn't start with a wildcard - Field = field2 where field2 is in a different table - Field op formula - Field IS NULL - Field IS NOT NULL - Field BETWEEN ... - Field IN ... - */ - - /* - We can't always use indexes when comparing a string index to a - number. cmp_type() is checked to allow compare of dates to numbers. - eq_func is NEVER true when num_values > 1 - */ - if (!eq_func) - { - /* - Additional optimization: if we're processing "t.key BETWEEN c1 AND c1" - then proceed as if we were processing "t.key = c1". - */ - if ((cond->functype() != Item_func::BETWEEN) || - ((Item_func_between*) cond)->negated || - !value[0]->eq(value[1], field->binary())) - return; - eq_func= TRUE; - } - - if (field->result_type() == STRING_RESULT) - { - if ((*value)->result_type() != STRING_RESULT) - { - if (field->cmp_type() != (*value)->result_type()) - return; - } - else - { - /* - We can't use indexes if the effective collation - of the operation differ from the field collation. - */ - if (field->cmp_type() == STRING_RESULT && - ((Field_str*)field)->charset() != cond->compare_collation()) - return; - } - } - - DBUG_ASSERT(eq_func); - /* Store possible eq field */ - (*eq_dep)->type= Module_dep::MODULE_EXPRESSION; //psergey-todo; - (*eq_dep)->field= get_field_value(te, field); - (*eq_dep)->expression= *value; - (*eq_dep)->level= and_level; - (*eq_dep)++; + if ((left->used_tables() & usable_tables) && + !(right->used_tables() & RAND_TABLE_BIT) && + left->real_item()->type() == Item::FIELD_ITEM) + { + Field *field= ((Item_field*)left->real_item())->field; + if (field->result_type() == STRING_RESULT) + { + if (right->result_type() != STRING_RESULT) + { + if (field->cmp_type() != right->result_type()) + return; + } + else + { + /* + We can't use indexes if the effective collation + of the operation differ from the field collation. + */ + if (field->cmp_type() == STRING_RESULT && + ((Field_str*)field)->charset() != cond->compare_collation()) + return; + } + } + + /* Store possible eq field */ + (*eq_dep)->type= Module_dep::MODULE_EXPRESSION; //psergey-todo; + (*eq_dep)->field= get_field_value(te, field); + (*eq_dep)->expression= right; + (*eq_dep)->level= and_level; + (*eq_dep)++; + } } @@ -1150,6 +1085,12 @@ { Outer_join_module *outer_join_dep= (Outer_join_module*)bound_modules; mark_as_eliminated(te->join, outer_join_dep->table_list); + if (!--te->n_outer_joins) + { + DBUG_PRINT("info", ("Table elimination eliminated everything" + " it theoretically could")); + return; + } break; } case Module_dep::MODULE_MULTI_EQUALITY:

1 0

[Maria-developers] Rev 2726: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r10/
by Sergey Petrunya 16 Aug '09

16 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r10/ ------------------------------------------------------------ revno: 2726 revision-id: psergey(a)askmonty.org-20090816091549-da84w3nlmx8prmvm parent: psergey(a)askmonty.org-20090816072524-w9fu2hy23pjwlr8z committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Sun 2009-08-16 12:15:49 +0300 message: MWL#17: Table elimination - code cleanup === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-16 07:25:24 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-16 09:15:49 +0000 @@ -125,7 +125,7 @@ make elements that depend on them bound, too. */ Module_dep *next; - uint unknown_args; /* TRUE<=> The entity is considered bound */ + uint unknown_args; Module_dep() : next(NULL), unknown_args(0) {} }; @@ -249,11 +249,9 @@ void eliminate_tables(JOIN *join); static void mark_as_eliminated(JOIN *join, TABLE_LIST *tbl); -#if 0 #ifndef DBUG_OFF static void dbug_print_deps(Table_elimination *te); #endif -#endif /*******************************************************************************************/ /* @@ -854,7 +852,7 @@ /* - This is used to analyse expressions in "tbl.col=expr" dependencies so + This is used to analyze expressions in "tbl.col=expr" dependencies so that we can figure out which fields the expression depends on. */ @@ -965,7 +963,7 @@ } *bound_deps_list= bound_dep; - //DBUG_EXECUTE("test", dbug_print_deps(te); ); + DBUG_EXECUTE("test", dbug_print_deps(te); ); DBUG_RETURN(FALSE); } @@ -1089,7 +1087,6 @@ void signal_from_field_to_exprs(Table_elimination* te, Field_value *field_dep, Module_dep **bound_modules) { - /* Now, expressions */ for (uint i=0; i < te->n_equality_deps; i++) { if (bitmap_is_set(&te->expr_deps, field_dep->bitmap_offset + i) && @@ -1213,7 +1210,6 @@ for (Outer_join_module *outer_join_dep= table_dep->outer_join_dep; outer_join_dep; outer_join_dep= outer_join_dep->parent) { - //if (!(outer_join_dep->missing_tables &= ~table_dep->table->map)) if (outer_join_dep->unknown_args && !--outer_join_dep->unknown_args) { @@ -1268,7 +1264,6 @@ } -#if 0 #ifndef DBUG_OFF static void dbug_print_deps(Table_elimination *te) @@ -1324,7 +1319,6 @@ } #endif -#endif /** @} (end of group Table_Elimination) */

1 0

[Maria-developers] Updated (by Psergey): Make --replicate-(do, ignore)-(db, table) behaviour for RBR identical to that of SBR (49)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Make --replicate-(do,ignore)-(db,table) behaviour for RBR identical to that of SBR CREATION DATE..: Sun, 16 Aug 2009, 12:01 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 49 (http://askmonty.org/worklog/?tid=49) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Sun, 16 Aug 2009, 12:06)=-=- High-Level Specification modified. --- /tmp/wklog.49.old.14928 2009-08-16 12:06:27.000000000 +0300 +++ /tmp/wklog.49.new.14928 2009-08-16 12:06:27.000000000 +0300 @@ -1 +1,12 @@ +Some notes: + +The only required changes are on the slave. The slave can see bounds between +statements (they are delimited by Query_event and Table_map_event entries), +Table_map_event lists all tables that are going to be updated => it is possible +to make a decision whether we should skip the statement, and if yes, skip all +RBR events that belong to the statement. + +Possible syntax for options +--replicate-wild-ignore-stmt-with-table=%.tmptbl% +--replicate-ignore-stmt-with-table=tmptbl -=-=(Psergey - Sun, 16 Aug 2009, 12:02)=-=- High Level Description modified. --- /tmp/wklog.49.old.14739 2009-08-16 12:02:05.000000000 +0300 +++ /tmp/wklog.49.new.14739 2009-08-16 12:02:05.000000000 +0300 @@ -6,8 +6,8 @@ This can be inconvenient, and also the semantics gets really complicated when --binlog_format=mixed is used. -This WL entry is about making processing of RBR events to work the same as SBR -events did. +This WL entry is about adding an option to make processing of RBR events to work +the same as SBR events did. DESCRIPTION: At the moment semantics of --replicate-(do,ignore)-(db,table) rules is different for RBR and SBR: http://dev.mysql.com/doc/refman/5.1/en/replication-rules-table-options.html This can be inconvenient, and also the semantics gets really complicated when --binlog_format=mixed is used. This WL entry is about adding an option to make processing of RBR events to work the same as SBR events did. HIGH-LEVEL SPECIFICATION: Some notes: The only required changes are on the slave. The slave can see bounds between statements (they are delimited by Query_event and Table_map_event entries), Table_map_event lists all tables that are going to be updated => it is possible to make a decision whether we should skip the statement, and if yes, skip all RBR events that belong to the statement. Possible syntax for options --replicate-wild-ignore-stmt-with-table=%.tmptbl% --replicate-ignore-stmt-with-table=tmptbl ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Make --replicate-(do, ignore)-(db, table) behaviour for RBR identical to that of SBR (49)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Make --replicate-(do,ignore)-(db,table) behaviour for RBR identical to that of SBR CREATION DATE..: Sun, 16 Aug 2009, 12:01 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 49 (http://askmonty.org/worklog/?tid=49) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Sun, 16 Aug 2009, 12:06)=-=- High-Level Specification modified. --- /tmp/wklog.49.old.14928 2009-08-16 12:06:27.000000000 +0300 +++ /tmp/wklog.49.new.14928 2009-08-16 12:06:27.000000000 +0300 @@ -1 +1,12 @@ +Some notes: + +The only required changes are on the slave. The slave can see bounds between +statements (they are delimited by Query_event and Table_map_event entries), +Table_map_event lists all tables that are going to be updated => it is possible +to make a decision whether we should skip the statement, and if yes, skip all +RBR events that belong to the statement. + +Possible syntax for options +--replicate-wild-ignore-stmt-with-table=%.tmptbl% +--replicate-ignore-stmt-with-table=tmptbl -=-=(Psergey - Sun, 16 Aug 2009, 12:02)=-=- High Level Description modified. --- /tmp/wklog.49.old.14739 2009-08-16 12:02:05.000000000 +0300 +++ /tmp/wklog.49.new.14739 2009-08-16 12:02:05.000000000 +0300 @@ -6,8 +6,8 @@ This can be inconvenient, and also the semantics gets really complicated when --binlog_format=mixed is used. -This WL entry is about making processing of RBR events to work the same as SBR -events did. +This WL entry is about adding an option to make processing of RBR events to work +the same as SBR events did. DESCRIPTION: At the moment semantics of --replicate-(do,ignore)-(db,table) rules is different for RBR and SBR: http://dev.mysql.com/doc/refman/5.1/en/replication-rules-table-options.html This can be inconvenient, and also the semantics gets really complicated when --binlog_format=mixed is used. This WL entry is about adding an option to make processing of RBR events to work the same as SBR events did. HIGH-LEVEL SPECIFICATION: Some notes: The only required changes are on the slave. The slave can see bounds between statements (they are delimited by Query_event and Table_map_event entries), Table_map_event lists all tables that are going to be updated => it is possible to make a decision whether we should skip the statement, and if yes, skip all RBR events that belong to the statement. Possible syntax for options --replicate-wild-ignore-stmt-with-table=%.tmptbl% --replicate-ignore-stmt-with-table=tmptbl ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Make --replicate-(do, ignore)-(db, table) behaviour for RBR identical to that of SBR (49)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Make --replicate-(do,ignore)-(db,table) behaviour for RBR identical to that of SBR CREATION DATE..: Sun, 16 Aug 2009, 12:01 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 49 (http://askmonty.org/worklog/?tid=49) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Sun, 16 Aug 2009, 12:02)=-=- High Level Description modified. --- /tmp/wklog.49.old.14739 2009-08-16 12:02:05.000000000 +0300 +++ /tmp/wklog.49.new.14739 2009-08-16 12:02:05.000000000 +0300 @@ -6,8 +6,8 @@ This can be inconvenient, and also the semantics gets really complicated when --binlog_format=mixed is used. -This WL entry is about making processing of RBR events to work the same as SBR -events did. +This WL entry is about adding an option to make processing of RBR events to work +the same as SBR events did. DESCRIPTION: At the moment semantics of --replicate-(do,ignore)-(db,table) rules is different for RBR and SBR: http://dev.mysql.com/doc/refman/5.1/en/replication-rules-table-options.html This can be inconvenient, and also the semantics gets really complicated when --binlog_format=mixed is used. This WL entry is about adding an option to make processing of RBR events to work the same as SBR events did. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Make --replicate-(do, ignore)-(db, table) behaviour for RBR identical to that of SBR (49)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Make --replicate-(do,ignore)-(db,table) behaviour for RBR identical to that of SBR CREATION DATE..: Sun, 16 Aug 2009, 12:01 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 49 (http://askmonty.org/worklog/?tid=49) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Sun, 16 Aug 2009, 12:02)=-=- High Level Description modified. --- /tmp/wklog.49.old.14739 2009-08-16 12:02:05.000000000 +0300 +++ /tmp/wklog.49.new.14739 2009-08-16 12:02:05.000000000 +0300 @@ -6,8 +6,8 @@ This can be inconvenient, and also the semantics gets really complicated when --binlog_format=mixed is used. -This WL entry is about making processing of RBR events to work the same as SBR -events did. +This WL entry is about adding an option to make processing of RBR events to work +the same as SBR events did. DESCRIPTION: At the moment semantics of --replicate-(do,ignore)-(db,table) rules is different for RBR and SBR: http://dev.mysql.com/doc/refman/5.1/en/replication-rules-table-options.html This can be inconvenient, and also the semantics gets really complicated when --binlog_format=mixed is used. This WL entry is about adding an option to make processing of RBR events to work the same as SBR events did. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Psergey): Make --replicate-(do, ignore)-(db, table) behaviour for RBR identical to that of SBR (49)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Make --replicate-(do,ignore)-(db,table) behaviour for RBR identical to that of SBR CREATION DATE..: Sun, 16 Aug 2009, 12:01 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 49 (http://askmonty.org/worklog/?tid=49) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: DESCRIPTION: At the moment semantics of --replicate-(do,ignore)-(db,table) rules is different for RBR and SBR: http://dev.mysql.com/doc/refman/5.1/en/replication-rules-table-options.html This can be inconvenient, and also the semantics gets really complicated when --binlog_format=mixed is used. This WL entry is about making processing of RBR events to work the same as SBR events did. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Psergey): Make --replicate-(do, ignore)-(db, table) behaviour for RBR identical to that of SBR (49)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Make --replicate-(do,ignore)-(db,table) behaviour for RBR identical to that of SBR CREATION DATE..: Sun, 16 Aug 2009, 12:01 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 49 (http://askmonty.org/worklog/?tid=49) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: DESCRIPTION: At the moment semantics of --replicate-(do,ignore)-(db,table) rules is different for RBR and SBR: http://dev.mysql.com/doc/refman/5.1/en/replication-rules-table-options.html This can be inconvenient, and also the semantics gets really complicated when --binlog_format=mixed is used. This WL entry is about making processing of RBR events to work the same as SBR events did. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Change BINLOG statement syntax to be human-readable (46)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Change BINLOG statement syntax to be human-readable CREATION DATE..: Sat, 15 Aug 2009, 23:42 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 46 (http://askmonty.org/worklog/?tid=46) VERSION........: WorkLog-3.4 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Sun, 16 Aug 2009, 11:30)=-=- High-Level Specification modified. --- /tmp/wklog.46.old.13453 2009-08-16 11:30:06.000000000 +0300 +++ /tmp/wklog.46.new.13453 2009-08-16 11:30:06.000000000 +0300 @@ -26,3 +26,7 @@ (due to locking, table open/close, etc). (TODO: is it really slower? we haven't checked). +* When SBR replication is used and the statements refer to the current database + (a common scenario), one can use awk to filter out updates made in certain + databases. The proposed syntax doesn't allow to perform equivalent filtering? + -=-=(Psergey - Sun, 16 Aug 2009, 11:13)=-=- High Level Description modified. --- /tmp/wklog.46.old.12747 2009-08-16 11:13:54.000000000 +0300 +++ /tmp/wklog.46.new.12747 2009-08-16 11:13:54.000000000 +0300 @@ -6,4 +6,4 @@ This WL task is about making BINLOG statements to be human-readable (either as an option or by default -The approach of this WL is to some extent an alternative to WL#38, WL#40, WL41. +The approach of this WL is to some extent an alternative to WL#38, WL#40, WL#41. -=-=(Psergey - Sun, 16 Aug 2009, 11:13)=-=- High Level Description modified. --- /tmp/wklog.46.old.12717 2009-08-16 11:13:40.000000000 +0300 +++ /tmp/wklog.46.new.12717 2009-08-16 11:13:40.000000000 +0300 @@ -5,3 +5,5 @@ This WL task is about making BINLOG statements to be human-readable (either as an option or by default + +The approach of this WL is to some extent an alternative to WL#38, WL#40, WL41. -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency deleted: 48 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency created: 48 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency deleted: 39 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sat, 15 Aug 2009, 23:43)=-=- High-Level Specification modified. --- /tmp/wklog.46.old.17742 2009-08-15 23:43:09.000000000 +0300 +++ /tmp/wklog.46.new.17742 2009-08-15 23:43:09.000000000 +0300 @@ -1 +1,28 @@ +Suggestion 1 +------------ +Original syntax suggestion by Kristian: + + BINLOG + WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 + TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 + TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 + WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 + UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), + FROM ('a') TO ('b') FLAGS 0x0 + DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; + + This is basically a dump of what is stored in the events, and would be an + alternative to BINLOG 'gwWEShMBAA...'. + +Feedback and other suggestions +------------------------------ +* What is the need for WITH TIMESTAMP part? Can't one use a separate + SET TIMESTAMP statement? + +* mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something + that's close to readable SQL. Can we make it to be regular parseable SQL? + + This will be syntax that's familiar to our parser and to the users + - A stream of SQL statements will be slower to run than BINLOG statements + (due to locking, table open/close, etc). (TODO: is it really slower? we + haven't checked). DESCRIPTION: One of great things about mysqlbinlog was that its output was human-readable SQL, so it was possible to edit it manually or with help of scripts. With RBR events and BINLOG 'DpiGShMBAAAALQAAADcBAA...' statements this is no longer the case. This WL task is about making BINLOG statements to be human-readable (either as an option or by default The approach of this WL is to some extent an alternative to WL#38, WL#40, WL#41. HIGH-LEVEL SPECIFICATION: Suggestion 1 ------------ Original syntax suggestion by Kristian: BINLOG WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), FROM ('a') TO ('b') FLAGS 0x0 DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; This is basically a dump of what is stored in the events, and would be an alternative to BINLOG 'gwWEShMBAA...'. Feedback and other suggestions ------------------------------ * What is the need for WITH TIMESTAMP part? Can't one use a separate SET TIMESTAMP statement? * mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something that's close to readable SQL. Can we make it to be regular parseable SQL? + This will be syntax that's familiar to our parser and to the users - A stream of SQL statements will be slower to run than BINLOG statements (due to locking, table open/close, etc). (TODO: is it really slower? we haven't checked). * When SBR replication is used and the statements refer to the current database (a common scenario), one can use awk to filter out updates made in certain databases. The proposed syntax doesn't allow to perform equivalent filtering? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Change BINLOG statement syntax to be human-readable (46)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Change BINLOG statement syntax to be human-readable CREATION DATE..: Sat, 15 Aug 2009, 23:42 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 46 (http://askmonty.org/worklog/?tid=46) VERSION........: WorkLog-3.4 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Sun, 16 Aug 2009, 11:30)=-=- High-Level Specification modified. --- /tmp/wklog.46.old.13453 2009-08-16 11:30:06.000000000 +0300 +++ /tmp/wklog.46.new.13453 2009-08-16 11:30:06.000000000 +0300 @@ -26,3 +26,7 @@ (due to locking, table open/close, etc). (TODO: is it really slower? we haven't checked). +* When SBR replication is used and the statements refer to the current database + (a common scenario), one can use awk to filter out updates made in certain + databases. The proposed syntax doesn't allow to perform equivalent filtering? + -=-=(Psergey - Sun, 16 Aug 2009, 11:13)=-=- High Level Description modified. --- /tmp/wklog.46.old.12747 2009-08-16 11:13:54.000000000 +0300 +++ /tmp/wklog.46.new.12747 2009-08-16 11:13:54.000000000 +0300 @@ -6,4 +6,4 @@ This WL task is about making BINLOG statements to be human-readable (either as an option or by default -The approach of this WL is to some extent an alternative to WL#38, WL#40, WL41. +The approach of this WL is to some extent an alternative to WL#38, WL#40, WL#41. -=-=(Psergey - Sun, 16 Aug 2009, 11:13)=-=- High Level Description modified. --- /tmp/wklog.46.old.12717 2009-08-16 11:13:40.000000000 +0300 +++ /tmp/wklog.46.new.12717 2009-08-16 11:13:40.000000000 +0300 @@ -5,3 +5,5 @@ This WL task is about making BINLOG statements to be human-readable (either as an option or by default + +The approach of this WL is to some extent an alternative to WL#38, WL#40, WL41. -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency deleted: 48 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency created: 48 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency deleted: 39 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sat, 15 Aug 2009, 23:43)=-=- High-Level Specification modified. --- /tmp/wklog.46.old.17742 2009-08-15 23:43:09.000000000 +0300 +++ /tmp/wklog.46.new.17742 2009-08-15 23:43:09.000000000 +0300 @@ -1 +1,28 @@ +Suggestion 1 +------------ +Original syntax suggestion by Kristian: + + BINLOG + WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 + TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 + TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 + WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 + UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), + FROM ('a') TO ('b') FLAGS 0x0 + DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; + + This is basically a dump of what is stored in the events, and would be an + alternative to BINLOG 'gwWEShMBAA...'. + +Feedback and other suggestions +------------------------------ +* What is the need for WITH TIMESTAMP part? Can't one use a separate + SET TIMESTAMP statement? + +* mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something + that's close to readable SQL. Can we make it to be regular parseable SQL? + + This will be syntax that's familiar to our parser and to the users + - A stream of SQL statements will be slower to run than BINLOG statements + (due to locking, table open/close, etc). (TODO: is it really slower? we + haven't checked). DESCRIPTION: One of great things about mysqlbinlog was that its output was human-readable SQL, so it was possible to edit it manually or with help of scripts. With RBR events and BINLOG 'DpiGShMBAAAALQAAADcBAA...' statements this is no longer the case. This WL task is about making BINLOG statements to be human-readable (either as an option or by default The approach of this WL is to some extent an alternative to WL#38, WL#40, WL#41. HIGH-LEVEL SPECIFICATION: Suggestion 1 ------------ Original syntax suggestion by Kristian: BINLOG WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), FROM ('a') TO ('b') FLAGS 0x0 DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; This is basically a dump of what is stored in the events, and would be an alternative to BINLOG 'gwWEShMBAA...'. Feedback and other suggestions ------------------------------ * What is the need for WITH TIMESTAMP part? Can't one use a separate SET TIMESTAMP statement? * mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something that's close to readable SQL. Can we make it to be regular parseable SQL? + This will be syntax that's familiar to our parser and to the users - A stream of SQL statements will be slower to run than BINLOG statements (due to locking, table open/close, etc). (TODO: is it really slower? we haven't checked). * When SBR replication is used and the statements refer to the current database (a common scenario), one can use awk to filter out updates made in certain databases. The proposed syntax doesn't allow to perform equivalent filtering? ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Change BINLOG statement syntax to be human-readable (46)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Change BINLOG statement syntax to be human-readable CREATION DATE..: Sat, 15 Aug 2009, 23:42 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 46 (http://askmonty.org/worklog/?tid=46) VERSION........: WorkLog-3.4 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Sun, 16 Aug 2009, 11:13)=-=- High Level Description modified. --- /tmp/wklog.46.old.12747 2009-08-16 11:13:54.000000000 +0300 +++ /tmp/wklog.46.new.12747 2009-08-16 11:13:54.000000000 +0300 @@ -6,4 +6,4 @@ This WL task is about making BINLOG statements to be human-readable (either as an option or by default -The approach of this WL is to some extent an alternative to WL#38, WL#40, WL41. +The approach of this WL is to some extent an alternative to WL#38, WL#40, WL#41. -=-=(Psergey - Sun, 16 Aug 2009, 11:13)=-=- High Level Description modified. --- /tmp/wklog.46.old.12717 2009-08-16 11:13:40.000000000 +0300 +++ /tmp/wklog.46.new.12717 2009-08-16 11:13:40.000000000 +0300 @@ -5,3 +5,5 @@ This WL task is about making BINLOG statements to be human-readable (either as an option or by default + +The approach of this WL is to some extent an alternative to WL#38, WL#40, WL41. -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency deleted: 48 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency created: 48 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency deleted: 39 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sat, 15 Aug 2009, 23:43)=-=- High-Level Specification modified. --- /tmp/wklog.46.old.17742 2009-08-15 23:43:09.000000000 +0300 +++ /tmp/wklog.46.new.17742 2009-08-15 23:43:09.000000000 +0300 @@ -1 +1,28 @@ +Suggestion 1 +------------ +Original syntax suggestion by Kristian: + + BINLOG + WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 + TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 + TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 + WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 + UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), + FROM ('a') TO ('b') FLAGS 0x0 + DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; + + This is basically a dump of what is stored in the events, and would be an + alternative to BINLOG 'gwWEShMBAA...'. + +Feedback and other suggestions +------------------------------ +* What is the need for WITH TIMESTAMP part? Can't one use a separate + SET TIMESTAMP statement? + +* mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something + that's close to readable SQL. Can we make it to be regular parseable SQL? + + This will be syntax that's familiar to our parser and to the users + - A stream of SQL statements will be slower to run than BINLOG statements + (due to locking, table open/close, etc). (TODO: is it really slower? we + haven't checked). DESCRIPTION: One of great things about mysqlbinlog was that its output was human-readable SQL, so it was possible to edit it manually or with help of scripts. With RBR events and BINLOG 'DpiGShMBAAAALQAAADcBAA...' statements this is no longer the case. This WL task is about making BINLOG statements to be human-readable (either as an option or by default The approach of this WL is to some extent an alternative to WL#38, WL#40, WL#41. HIGH-LEVEL SPECIFICATION: Suggestion 1 ------------ Original syntax suggestion by Kristian: BINLOG WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), FROM ('a') TO ('b') FLAGS 0x0 DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; This is basically a dump of what is stored in the events, and would be an alternative to BINLOG 'gwWEShMBAA...'. Feedback and other suggestions ------------------------------ * What is the need for WITH TIMESTAMP part? Can't one use a separate SET TIMESTAMP statement? * mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something that's close to readable SQL. Can we make it to be regular parseable SQL? + This will be syntax that's familiar to our parser and to the users - A stream of SQL statements will be slower to run than BINLOG statements (due to locking, table open/close, etc). (TODO: is it really slower? we haven't checked). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Change BINLOG statement syntax to be human-readable (46)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Change BINLOG statement syntax to be human-readable CREATION DATE..: Sat, 15 Aug 2009, 23:42 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 46 (http://askmonty.org/worklog/?tid=46) VERSION........: WorkLog-3.4 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Sun, 16 Aug 2009, 11:13)=-=- High Level Description modified. --- /tmp/wklog.46.old.12747 2009-08-16 11:13:54.000000000 +0300 +++ /tmp/wklog.46.new.12747 2009-08-16 11:13:54.000000000 +0300 @@ -6,4 +6,4 @@ This WL task is about making BINLOG statements to be human-readable (either as an option or by default -The approach of this WL is to some extent an alternative to WL#38, WL#40, WL41. +The approach of this WL is to some extent an alternative to WL#38, WL#40, WL#41. -=-=(Psergey - Sun, 16 Aug 2009, 11:13)=-=- High Level Description modified. --- /tmp/wklog.46.old.12717 2009-08-16 11:13:40.000000000 +0300 +++ /tmp/wklog.46.new.12717 2009-08-16 11:13:40.000000000 +0300 @@ -5,3 +5,5 @@ This WL task is about making BINLOG statements to be human-readable (either as an option or by default + +The approach of this WL is to some extent an alternative to WL#38, WL#40, WL41. -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency deleted: 48 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency created: 48 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency deleted: 39 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sat, 15 Aug 2009, 23:43)=-=- High-Level Specification modified. --- /tmp/wklog.46.old.17742 2009-08-15 23:43:09.000000000 +0300 +++ /tmp/wklog.46.new.17742 2009-08-15 23:43:09.000000000 +0300 @@ -1 +1,28 @@ +Suggestion 1 +------------ +Original syntax suggestion by Kristian: + + BINLOG + WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 + TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 + TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 + WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 + UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), + FROM ('a') TO ('b') FLAGS 0x0 + DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; + + This is basically a dump of what is stored in the events, and would be an + alternative to BINLOG 'gwWEShMBAA...'. + +Feedback and other suggestions +------------------------------ +* What is the need for WITH TIMESTAMP part? Can't one use a separate + SET TIMESTAMP statement? + +* mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something + that's close to readable SQL. Can we make it to be regular parseable SQL? + + This will be syntax that's familiar to our parser and to the users + - A stream of SQL statements will be slower to run than BINLOG statements + (due to locking, table open/close, etc). (TODO: is it really slower? we + haven't checked). DESCRIPTION: One of great things about mysqlbinlog was that its output was human-readable SQL, so it was possible to edit it manually or with help of scripts. With RBR events and BINLOG 'DpiGShMBAAAALQAAADcBAA...' statements this is no longer the case. This WL task is about making BINLOG statements to be human-readable (either as an option or by default The approach of this WL is to some extent an alternative to WL#38, WL#40, WL#41. HIGH-LEVEL SPECIFICATION: Suggestion 1 ------------ Original syntax suggestion by Kristian: BINLOG WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), FROM ('a') TO ('b') FLAGS 0x0 DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; This is basically a dump of what is stored in the events, and would be an alternative to BINLOG 'gwWEShMBAA...'. Feedback and other suggestions ------------------------------ * What is the need for WITH TIMESTAMP part? Can't one use a separate SET TIMESTAMP statement? * mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something that's close to readable SQL. Can we make it to be regular parseable SQL? + This will be syntax that's familiar to our parser and to the users - A stream of SQL statements will be slower to run than BINLOG statements (due to locking, table open/close, etc). (TODO: is it really slower? we haven't checked). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Change BINLOG statement syntax to be human-readable (46)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Change BINLOG statement syntax to be human-readable CREATION DATE..: Sat, 15 Aug 2009, 23:42 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 46 (http://askmonty.org/worklog/?tid=46) VERSION........: WorkLog-3.4 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Sun, 16 Aug 2009, 11:13)=-=- High Level Description modified. --- /tmp/wklog.46.old.12717 2009-08-16 11:13:40.000000000 +0300 +++ /tmp/wklog.46.new.12717 2009-08-16 11:13:40.000000000 +0300 @@ -5,3 +5,5 @@ This WL task is about making BINLOG statements to be human-readable (either as an option or by default + +The approach of this WL is to some extent an alternative to WL#38, WL#40, WL41. -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency deleted: 48 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency created: 48 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency deleted: 39 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sat, 15 Aug 2009, 23:43)=-=- High-Level Specification modified. --- /tmp/wklog.46.old.17742 2009-08-15 23:43:09.000000000 +0300 +++ /tmp/wklog.46.new.17742 2009-08-15 23:43:09.000000000 +0300 @@ -1 +1,28 @@ +Suggestion 1 +------------ +Original syntax suggestion by Kristian: + + BINLOG + WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 + TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 + TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 + WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 + UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), + FROM ('a') TO ('b') FLAGS 0x0 + DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; + + This is basically a dump of what is stored in the events, and would be an + alternative to BINLOG 'gwWEShMBAA...'. + +Feedback and other suggestions +------------------------------ +* What is the need for WITH TIMESTAMP part? Can't one use a separate + SET TIMESTAMP statement? + +* mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something + that's close to readable SQL. Can we make it to be regular parseable SQL? + + This will be syntax that's familiar to our parser and to the users + - A stream of SQL statements will be slower to run than BINLOG statements + (due to locking, table open/close, etc). (TODO: is it really slower? we + haven't checked). DESCRIPTION: One of great things about mysqlbinlog was that its output was human-readable SQL, so it was possible to edit it manually or with help of scripts. With RBR events and BINLOG 'DpiGShMBAAAALQAAADcBAA...' statements this is no longer the case. This WL task is about making BINLOG statements to be human-readable (either as an option or by default The approach of this WL is to some extent an alternative to WL#38, WL#40, WL41. HIGH-LEVEL SPECIFICATION: Suggestion 1 ------------ Original syntax suggestion by Kristian: BINLOG WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), FROM ('a') TO ('b') FLAGS 0x0 DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; This is basically a dump of what is stored in the events, and would be an alternative to BINLOG 'gwWEShMBAA...'. Feedback and other suggestions ------------------------------ * What is the need for WITH TIMESTAMP part? Can't one use a separate SET TIMESTAMP statement? * mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something that's close to readable SQL. Can we make it to be regular parseable SQL? + This will be syntax that's familiar to our parser and to the users - A stream of SQL statements will be slower to run than BINLOG statements (due to locking, table open/close, etc). (TODO: is it really slower? we haven't checked). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Change BINLOG statement syntax to be human-readable (46)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Change BINLOG statement syntax to be human-readable CREATION DATE..: Sat, 15 Aug 2009, 23:42 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 46 (http://askmonty.org/worklog/?tid=46) VERSION........: WorkLog-3.4 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Sun, 16 Aug 2009, 11:13)=-=- High Level Description modified. --- /tmp/wklog.46.old.12717 2009-08-16 11:13:40.000000000 +0300 +++ /tmp/wklog.46.new.12717 2009-08-16 11:13:40.000000000 +0300 @@ -5,3 +5,5 @@ This WL task is about making BINLOG statements to be human-readable (either as an option or by default + +The approach of this WL is to some extent an alternative to WL#38, WL#40, WL41. -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 11:07)=-=- Dependency deleted: 48 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency created: 48 now depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 10:59)=-=- Dependency deleted: 39 no longer depends on 46 -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 46 -=-=(Psergey - Sat, 15 Aug 2009, 23:43)=-=- High-Level Specification modified. --- /tmp/wklog.46.old.17742 2009-08-15 23:43:09.000000000 +0300 +++ /tmp/wklog.46.new.17742 2009-08-15 23:43:09.000000000 +0300 @@ -1 +1,28 @@ +Suggestion 1 +------------ +Original syntax suggestion by Kristian: + + BINLOG + WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 + TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 + TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 + WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 + UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), + FROM ('a') TO ('b') FLAGS 0x0 + DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; + + This is basically a dump of what is stored in the events, and would be an + alternative to BINLOG 'gwWEShMBAA...'. + +Feedback and other suggestions +------------------------------ +* What is the need for WITH TIMESTAMP part? Can't one use a separate + SET TIMESTAMP statement? + +* mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something + that's close to readable SQL. Can we make it to be regular parseable SQL? + + This will be syntax that's familiar to our parser and to the users + - A stream of SQL statements will be slower to run than BINLOG statements + (due to locking, table open/close, etc). (TODO: is it really slower? we + haven't checked). DESCRIPTION: One of great things about mysqlbinlog was that its output was human-readable SQL, so it was possible to edit it manually or with help of scripts. With RBR events and BINLOG 'DpiGShMBAAAALQAAADcBAA...' statements this is no longer the case. This WL task is about making BINLOG statements to be human-readable (either as an option or by default The approach of this WL is to some extent an alternative to WL#38, WL#40, WL41. HIGH-LEVEL SPECIFICATION: Suggestion 1 ------------ Original syntax suggestion by Kristian: BINLOG WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), FROM ('a') TO ('b') FLAGS 0x0 DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; This is basically a dump of what is stored in the events, and would be an alternative to BINLOG 'gwWEShMBAA...'. Feedback and other suggestions ------------------------------ * What is the need for WITH TIMESTAMP part? Can't one use a separate SET TIMESTAMP statement? * mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something that's close to readable SQL. Can we make it to be regular parseable SQL? + This will be syntax that's familiar to our parser and to the users - A stream of SQL statements will be slower to run than BINLOG statements (due to locking, table open/close, etc). (TODO: is it really slower? we haven't checked). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Store in binlog text of statements that caused RBR events (47)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Store in binlog text of statements that caused RBR events CREATION DATE..: Sat, 15 Aug 2009, 23:48 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 47 (http://askmonty.org/worklog/?tid=47) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Sun, 16 Aug 2009, 11:08)=-=- High-Level Specification modified. --- /tmp/wklog.47.old.12485 2009-08-16 11:08:33.000000000 +0300 +++ /tmp/wklog.47.new.12485 2009-08-16 11:08:33.000000000 +0300 @@ -1 +1,6 @@ +First suggestion: + +> I think for this we would actually need a new binlog event type +> (Comment_log_event?). Unless we want to log an empty statement Query_log_event +> containing only a comment (a bit of a hack). -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 47 DESCRIPTION: Store in binlog (and show in mysqlbinlog output) texts of statements that caused RBR events This is needed for (list from Monty): - Easier to understand why updates happened - Would make it easier to find out where in application things went wrong (as you can search for exact strings) - Allow one to filter things based on comments in the statement. The cost of this can be that the binlog will be approximately 2x in size (especially insert of big blob's would be a bit painful), so this should be an optional feature. HIGH-LEVEL SPECIFICATION: First suggestion: > I think for this we would actually need a new binlog event type > (Comment_log_event?). Unless we want to log an empty statement Query_log_event > containing only a comment (a bit of a hack). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Store in binlog text of statements that caused RBR events (47)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Store in binlog text of statements that caused RBR events CREATION DATE..: Sat, 15 Aug 2009, 23:48 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 47 (http://askmonty.org/worklog/?tid=47) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Sun, 16 Aug 2009, 11:08)=-=- High-Level Specification modified. --- /tmp/wklog.47.old.12485 2009-08-16 11:08:33.000000000 +0300 +++ /tmp/wklog.47.new.12485 2009-08-16 11:08:33.000000000 +0300 @@ -1 +1,6 @@ +First suggestion: + +> I think for this we would actually need a new binlog event type +> (Comment_log_event?). Unless we want to log an empty statement Query_log_event +> containing only a comment (a bit of a hack). -=-=(Psergey - Sun, 16 Aug 2009, 00:02)=-=- Dependency created: 39 now depends on 47 DESCRIPTION: Store in binlog (and show in mysqlbinlog output) texts of statements that caused RBR events This is needed for (list from Monty): - Easier to understand why updates happened - Would make it easier to find out where in application things went wrong (as you can search for exact strings) - Allow one to filter things based on comments in the statement. The cost of this can be that the binlog will be approximately 2x in size (especially insert of big blob's would be a bit painful), so this should be an optional feature. HIGH-LEVEL SPECIFICATION: First suggestion: > I think for this we would actually need a new binlog event type > (Comment_log_event?). Unless we want to log an empty statement Query_log_event > containing only a comment (a bit of a hack). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Psergey): Extra replication tasks (48)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Extra replication tasks CREATION DATE..: Sun, 16 Aug 2009, 10:58 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 48 (http://askmonty.org/worklog/?tid=48) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: DESCRIPTION: An umbrella task for replication tasks that are nice to do but are not direct responses for customer requests ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Psergey): Extra replication tasks (48)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Extra replication tasks CREATION DATE..: Sun, 16 Aug 2009, 10:58 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 48 (http://askmonty.org/worklog/?tid=48) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: DESCRIPTION: An umbrella task for replication tasks that are nice to do but are not direct responses for customer requests ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Rev 2725: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r10-vg/
by Sergey Petrunya 16 Aug '09

16 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r10-vg/ ------------------------------------------------------------ revno: 2725 revision-id: psergey(a)askmonty.org-20090816072524-w9fu2hy23pjwlr8z parent: psergey(a)askmonty.org-20090815153912-q47vfp1j22ilmup2 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10-vg timestamp: Sun 2009-08-16 10:25:24 +0300 message: MWL#17: Table elimination - Fix trivial valgrind failures that shown up after review === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-15 15:39:12 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-16 07:25:24 +0000 @@ -40,13 +40,16 @@ Table elimination is redone on every PS re-execution. */ -class Value_dep +class Value_dep : public Sql_alloc { public: enum { VALUE_FIELD, VALUE_TABLE, } type; /* Type of the object */ + + Value_dep(): bound(FALSE), next(NULL) + {} bool bound; Value_dep *next;

1 0

[Maria-developers] Updated (by Guest): index_merge: fair choice between index_merge union and range access (24)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: index_merge: fair choice between index_merge union and range access CREATION DATE..: Tue, 26 May 2009, 12:10 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: Psergey CATEGORY.......: Server-Sprint TASK ID........: 24 (http://askmonty.org/worklog/?tid=24) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Sun, 16 Aug 2009, 02:13)=-=- Low Level Design modified. --- /tmp/wklog.24.old.23383 2009-08-16 02:13:54.000000000 +0300 +++ /tmp/wklog.24.new.23383 2009-08-16 02:13:54.000000000 +0300 @@ -125,7 +125,7 @@ The optimizer will generate the plans that only use the "col1=c1" part. The right side of the AND will be ignored even if it has good selectivity. -(Here an imerge for col2=c2 OR col3=c3 won't be built since neither col2=c2 nor +(Here no imerge for col2=c2 OR col3=c3 will be built since neither col2=c2 nor col3=c3 represent index ranges.) @@ -199,7 +199,7 @@ O2. "Create index_merge accesses when possible" Current tree_or() will not create index_merge access when it could create - non-index merge access (see DISCARD-IMERGE-3 and its example in the "Problems + non-index merge access (see DISCARD-IMERGE-2 and its example in the "Problems in the current implementation" section). This will be changed to work as follows: we will create index_merge made for index scans that didn't have their match in the other sel_tree. -=-=(Guest - Sun, 16 Aug 2009, 01:03)=-=- Low Level Design modified. --- /tmp/wklog.24.old.20767 2009-08-16 01:03:11.000000000 +0300 +++ /tmp/wklog.24.new.20767 2009-08-16 01:03:11.000000000 +0300 @@ -18,6 +18,8 @@ # a range tree has range access options, possibly for several keys range_tree = range(key1) AND range(key2) AND ... AND range(keyN); + (here range(keyi) may represent ranges not for initial keyi prefixes, + but ranges for any infixes for keyi) # merge tree represents several way to index_merge imerge_tree = imerge1 AND imerge2 AND ... @@ -47,13 +49,13 @@ R.add(range_union(A.range(i), B.range(i))); if (R has at least one range access) - return R; + return R; // DISCARD-IMERGE-2 else { /* could not build any range accesses. construct index_merge */ - remove non-ranges from A; // DISCARD-IMERGE-2 + remove non-ranges from A; remove non-ranges from B; - return new index_merge(A, B); + return new index_merge(A, B); // DISCARD-IMERGE-3 } } else if (A is range tree and B is index_merge tree (or vice versa)) @@ -65,12 +67,12 @@ (range_treeB_11 OR range_treeB_12 OR ... OR range_treeB_1N) AND (range_treeB_21 OR range_treeB_22 OR ... OR range_treeB_2N) AND ... - (range_treeB_K1 OR range_treeB_K2 OR ... OR range_treeB_kN) AND + (range_treeB_K1 OR range_treeB_K2 OR ... OR range_treeB_kN) = (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND (range_treeA OR range_treeB_21 OR ... OR range_treeB_2N) AND ... - (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND + (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) Now each line represents an index_merge.. } @@ -82,18 +84,18 @@ OR imergeB1 AND imergeB2 AND ... AND imergeBN - -> (discard all imergeA{i=2,3,...} -> // DISCARD-IMERGE-3 + -> (discard all imergeA{i=2,3,...} -> // DISCARD-IMERGE-4 imergeA1 OR - imergeB1 AND imergeB2 AND ... AND imergeBN = + imergeB1 = - = (combine imergeA1 with each of the imergeB{i} ) = + = (combine imergeA1 with each of the range_treeB_1{i} ) = - combine(imergeA1 OR imergeB1) AND - combine(imergeA1 OR imergeB2) AND + combine(imergeA1 OR range_treeB_11) AND + combine(imergeA1 OR range_treeB_12) AND ... AND - combine(imergeA1 OR imergeBN) + combine(imergeA1 OR range_treeB_1N) } } @@ -109,7 +111,7 @@ DISCARD-IMERGE-2 step will cause index_merge option to be discarded when the WHERE clause has this form (conditions t.badkey may have abritrary form): - (t.badkey<c1 AND t.key1=c1) OR (t.key1=c2 AND t.badkey < c2) + (t.badkey<c1 AND t.key1=c1) OR (t.key2=c2 AND t.badkey < c2) DISCARD-IMERGE-3 manifests itself as the following effect: suppose there are two indexes: @@ -123,6 +125,8 @@ The optimizer will generate the plans that only use the "col1=c1" part. The right side of the AND will be ignored even if it has good selectivity. +(Here an imerge for col2=c2 OR col3=c3 won't be built since neither col2=c2 nor +col3=c3 represent index ranges.) 2. New implementation -=-=(Guest - Mon, 20 Jul 2009, 17:13)=-=- Dependency deleted: 30 no longer depends on 24 -=-=(Guest - Sat, 20 Jun 2009, 09:34)=-=- Low Level Design modified. --- /tmp/wklog.24.old.21663 2009-06-20 09:34:48.000000000 +0300 +++ /tmp/wklog.24.new.21663 2009-06-20 09:34:48.000000000 +0300 @@ -4,6 +4,7 @@ 2. New implementation 2.1 New tree_and() 2.2 New tree_or() +3. Testing and required coverage </contents> 1. Current implementation overview @@ -240,3 +241,14 @@ In order to limit the impact of this combinatorial explosion, we will introduce a rule that we won't generate more than #defined MAX_IMERGE_OPTS options. + +3. Testing and required coverage +================================ +So far could find the following user cases: + +* BUG#17259: Query optimizer chooses wrong index +* BUG#17673: Optimizer does not use Index Merge optimization in some cases +* BUG#23322: Optimizer sometimes erroniously prefers other index over index merge +* BUG#30151: optimizer is very reluctant to chose index_merge algorithm + + -=-=(Guest - Thu, 18 Jun 2009, 16:55)=-=- Low Level Design modified. --- /tmp/wklog.24.old.19152 2009-06-18 16:55:00.000000000 +0300 +++ /tmp/wklog.24.new.19152 2009-06-18 16:55:00.000000000 +0300 @@ -141,13 +141,15 @@ Operations on SEL_ARG trees will be modified to produce/process the trees of this kind: + 2.1 New tree_and() ------------------ In order not to lose plans, we'll make these changes: -1. Don't remove index_merge part of the tree. +A1. Don't remove index_merge part of the tree (this will take care of + DISCARD-IMERGE-1 problem) -2. Push range conditions down into index_merge trees that may support them. +A2. Push range conditions down into index_merge trees that may support them. if one tree has range(key1) and the other tree has imerge(key1 OR key2) then perform an equvalent of this operation: @@ -155,8 +157,86 @@ (rangeA(key1) AND rangeB(key1)) OR (rangeA(key1) AND rangeB(key2)) -3. Just as before: if both sel_tree A and sel_tree B have index_merge options, +A3. Just as before: if both sel_tree A and sel_tree B have index_merge options, concatenate them together. -2.2 New tree_or() +2.2 New tree_or() +----------------- +O1. Dont remove non-range plans: + Current tree_or() code will refuse to produce index_merge plans for + conditions like + + "t.key1part2=const OR t.key2part1=const" + + (this is marked as DISCARD-IMERGE-3). This was justifed as the left part of + the AND condition is not usable for range access, and the operation of + tree_and() guaranteed that there was no way it could changed to make a + usable range plan. With new tree_and() and rule A2, this is no longer the + case. For example for this query: + + (t.key1part2=const OR t.key2part1=const) AND t.key1part1=const + + it will construct a + + imerge(t.key1part2=const OR t.key2part1=const), range(t.key1part1=const) + + then tree_and() will apply rule A2 to push the range down into index merge + and after that we'll have: + + range(t.key1part1=const) + imerge( + t.key1part2=const AND t.key1part1=const, + t.key2part1=const + ) + note that imerge(...) describes a usable index_merge plan and it's possible + that it will be the best access path. + +O2. "Create index_merge accesses when possible" + Current tree_or() will not create index_merge access when it could create + non-index merge access (see DISCARD-IMERGE-3 and its example in the "Problems + in the current implementation" section). This will be changed to work as + follows: we will create index_merge made for index scans that didn't have + their match in the other sel_tree. + Ilustrating it with an example: + + | sel_tree_A | sel_tree_B | A or B | include in index_merge? + ------+------------+------------+--------+------------------------ + key1 | cond1 | cond2 | condM | no + key2 | cond3 | cond4 | NULL | no + key3 | cond5 | | | yes, A-side + key4 | cond6 | | | yes, A-side + key5 | | cond7 | | yes, B-side + key6 | | cond8 | | yes, B-side + + here we assume that + - (cond1 OR cond2) did produce a combined range. Not including them in + index_merge. + - (cond3 OR cond4) didn't produce a usable range (e.g. they were + t.key1part1=c1 AND t.key1part2=c1, respectively, and combining them + didn't yield any range list) + - All other scand didn't have their counterparts, so we'll end up with a + SEL_TREE of: + + range(condM) AND index_merge((cond5 AND cond6),(cond7 AND cond8)) + . + +O4. There is no O4. DISCARD-INDEX-MERGE-4 will remain there. The idea is +that although DISCARD-INDEX-MERGE-4 does discard plans, so far we haven +seen any complaints that could be attributed to it. +If we face the need to lift DISCARD-INDEX-MERGE-4, our answer will be to +lift it ,and produce a cross-product: + + ((key1p OR key2p) AND (key3p OR key4p)) + OR + ((key5p OR key6p) AND (key7p OR key8p)) + + = (key1p OR key2p OR key5p OR key6p) AND // this part is currently + (key3p OR key4p OR key5p OR key6p) AND // produced + + (key1p OR key2p OR key5p OR key6p) AND // this part will be added + (key3p OR key4p OR key5p OR key6p) //. + +In order to limit the impact of this combinatorial explosion, we will +introduce a rule that we won't generate more than #defined +MAX_IMERGE_OPTS options. -=-=(Guest - Thu, 18 Jun 2009, 14:56)=-=- Low Level Design modified. --- /tmp/wklog.24.old.15612 2009-06-18 14:56:09.000000000 +0300 +++ /tmp/wklog.24.new.15612 2009-06-18 14:56:09.000000000 +0300 @@ -1 +1,162 @@ +<contents> +1. Current implementation overview +1.1. Problems in the current implementation +2. New implementation +2.1 New tree_and() +2.2 New tree_or() +</contents> + +1. Current implementation overview +================================== +At the moment, range analyzer works as follows: + +SEL_TREE structure represents + + # There are sel_trees, a sel_tree is either range or merge tree + sel_tree = range_tree | imerge_tree + + # a range tree has range access options, possibly for several keys + range_tree = range(key1) AND range(key2) AND ... AND range(keyN); + + # merge tree represents several way to index_merge + imerge_tree = imerge1 AND imerge2 AND ... + + # a way to do index merge == a set to use of different indexes. + imergeX = range_tree1 OR range_tree2 OR .. + where no pair of range_treeX have ranges over the same index. + + + tree_and(A, B) + { + if (both A and B are range trees) + return a range_tree with computed intersection for each range; + if (only one of A and B is a range tree) + return that tree; // DISCARD-IMERGE-1 + // at this point both trees are index_merge trees + return concat_lists( A.imerge1 ... A.imergeN, B.imerge1 ... B.imergeN); + } + + + tree_or(A, B) + { + if (A and B are range trees) + { + R = new range_tree; + for each index i + R.add(range_union(A.range(i), B.range(i))); + + if (R has at least one range access) + return R; + else + { + /* could not build any range accesses. construct index_merge */ + remove non-ranges from A; // DISCARD-IMERGE-2 + remove non-ranges from B; + return new index_merge(A, B); + } + } + else if (A is range tree and B is index_merge tree (or vice versa)) + { + Perform this transformation: + + range_treeA // this is A + OR + (range_treeB_11 OR range_treeB_12 OR ... OR range_treeB_1N) AND + (range_treeB_21 OR range_treeB_22 OR ... OR range_treeB_2N) AND + ... + (range_treeB_K1 OR range_treeB_K2 OR ... OR range_treeB_kN) AND + = + (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND + (range_treeA OR range_treeB_21 OR ... OR range_treeB_2N) AND + ... + (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND + + Now each line represents an index_merge.. + } + else if (both A and B are index_merge trees) + { + Perform this transformation: + + imergeA1 AND imergeA2 AND ... AND imergeAN + OR + imergeB1 AND imergeB2 AND ... AND imergeBN + + -> (discard all imergeA{i=2,3,...} -> // DISCARD-IMERGE-3 + + imergeA1 + OR + imergeB1 AND imergeB2 AND ... AND imergeBN = + + = (combine imergeA1 with each of the imergeB{i} ) = + + combine(imergeA1 OR imergeB1) AND + combine(imergeA1 OR imergeB2) AND + ... AND + combine(imergeA1 OR imergeBN) + } + } + +1.1. Problems in the current implementation +------------------------------------------- +As marked in the code above: + +DISCARD-IMERGE-1 step will cause index_merge option to be discarded when +the WHERE clause has this form: + + (t.key1=c1 OR t.key2=c2) AND t.badkey < c3 + +DISCARD-IMERGE-2 step will cause index_merge option to be discarded when +the WHERE clause has this form (conditions t.badkey may have abritrary form): + + (t.badkey<c1 AND t.key1=c1) OR (t.key1=c2 AND t.badkey < c2) + +DISCARD-IMERGE-3 manifests itself as the following effect: suppose there are +two indexes: + + INDEX i1(col1, col2), + INDEX i2(col1, col3) + +and this WHERE clause: + + col1=c1 AND (col2=c2 OR col3=c3) + +The optimizer will generate the plans that only use the "col1=c1" part. The +right side of the AND will be ignored even if it has good selectivity. + + +2. New implementation +===================== + +<general idea> +* Don't start fighting combinatorial explosion until we've actually got one. +</> + +SEL_TREE structure will be now able to hold both index_merge and range scan +candidates at the same time. That is, + + sel_tree2 = range_tree AND imerge_tree + +where both parts are optional (i.e. can be empty) + +Operations on SEL_ARG trees will be modified to produce/process the trees of +this kind: + +2.1 New tree_and() +------------------ +In order not to lose plans, we'll make these changes: + +1. Don't remove index_merge part of the tree. + +2. Push range conditions down into index_merge trees that may support them. + if one tree has range(key1) and the other tree has imerge(key1 OR key2) + then perform an equvalent of this operation: + + rangeA(key1) AND ( rangeB(key1) OR rangeB(key2)) = + + (rangeA(key1) AND rangeB(key1)) OR (rangeA(key1) AND rangeB(key2)) + +3. Just as before: if both sel_tree A and sel_tree B have index_merge options, + concatenate them together. + +2.2 New tree_or() -=-=(Psergey - Wed, 03 Jun 2009, 12:09)=-=- Dependency created: 30 now depends on 24 -=-=(Guest - Mon, 01 Jun 2009, 23:30)=-=- High-Level Specification modified. --- /tmp/wklog.24.old.21580 2009-06-01 23:30:06.000000000 +0300 +++ /tmp/wklog.24.new.21580 2009-06-01 23:30:06.000000000 +0300 @@ -64,6 +64,9 @@ * How strict is the limitation on the form of the WHERE? +* Which version should this be based on? 5.1? Which patches are should be in + (google's/percona's/maria/etc?) + * TODO: The optimizer didn't compare costs of index_merge and range before (ok it did but that was done for accesses to different tables). Will there be any possible gotchas here? -=-=(Guest - Wed, 27 May 2009, 13:59)=-=- Title modified. --- /tmp/wklog.24.old.9498 2009-05-27 13:59:23.000000000 +0300 +++ /tmp/wklog.24.new.9498 2009-05-27 13:59:23.000000000 +0300 @@ -1 +1 @@ -index_merge optimizer: dont discard index_merge union strategies when range is available +index_merge: fair choice between index_merge union and range access -=-=(Guest - Tue, 26 May 2009, 13:27)=-=- High-Level Specification modified. --- /tmp/wklog.24.old.305 2009-05-26 13:27:32.000000000 +0300 +++ /tmp/wklog.24.new.305 2009-05-26 13:27:32.000000000 +0300 @@ -1 +1,70 @@ +(Not a ready HLS but draft) +<contents> +Solution overview +Limitations +TODO + +</contents> + +Solution overview +================= +The idea is to delay discarding potential index_merge plans until the point +where it is really necessary. + +This way, we won't have to do much changes in the range analyzer, but will be +able to keep potential index_merge plan just enough so that it's possible to +take it into consideration together with range access plans. + +Since there are no changes in the optimizer, the ability to consider both +range and index_merge options will be limited to WHERE clauses of this form: + + WHERE := range_cond(key1_1) AND + range_cond(key2_1) AND + other_cond AND + index_merge_OR_cond1(key3_1, key3_2, ...) + index_merge_OR_cond2(key4_1, key4_2, ...) + +where + + index_merge_OR_cond{N} := (range_cond(keyN_1) OR + range_cond(keyN_2) OR ...) + + + range_cond(keyX) := condition that allows to construct range access of keyX + and doesn't allow to construct range/index_merge accesses + for any keys of the table in question. + + +For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: + + SEL_TREE( + range(key1_1), + ... + range(key2_1), + SEL_IMERGE( (1) + SEL_TREE(key3_1}) + SEL_TREE(key3_2}) + ... + ) + ... + ) + +which can be used to make a cost-based choice between range and index_merge. + +Limitations +----------- +This will not be a full solution in a sense that the range analyzer will not +be able to produce sel_tree (1) if the WHERE clause is specified in other form +(e.g. brackets were opened). + +TODO +---- +* is it a problem if there are keys that are referred to both from + index_merge and from range access? + +* How strict is the limitation on the form of the WHERE? + +* TODO: The optimizer didn't compare costs of index_merge and range before (ok + it did but that was done for accesses to different tables). Will there be any + possible gotchas here? DESCRIPTION: Current range optimizer will discard possible index_merge/[sort]union strategies when there is a possible range plan. This action is a part of measures we take to avoid combinatorial explosion of possible range/ index_merge strategies. A bad side effect of this is that for WHERE clauses in form t.key1= 'very-frequent-value' AND (t.key2='rare-value1' OR t.key3='rare-value2') the optimizer will - discard union(key2,key3) in favor of range(key1) - consider costs of using range(key1) and discard that plan also and the overall effect is that possible poor range access will cause possible good index_merge access not to be considered. This WL is to about lifting this limitation at least for some subset of WHERE clauses. HIGH-LEVEL SPECIFICATION: (Not a ready HLS but draft) <contents> Solution overview Limitations TODO </contents> Solution overview ================= The idea is to delay discarding potential index_merge plans until the point where it is really necessary. This way, we won't have to do much changes in the range analyzer, but will be able to keep potential index_merge plan just enough so that it's possible to take it into consideration together with range access plans. Since there are no changes in the optimizer, the ability to consider both range and index_merge options will be limited to WHERE clauses of this form: WHERE := range_cond(key1_1) AND range_cond(key2_1) AND other_cond AND index_merge_OR_cond1(key3_1, key3_2, ...) index_merge_OR_cond2(key4_1, key4_2, ...) where index_merge_OR_cond{N} := (range_cond(keyN_1) OR range_cond(keyN_2) OR ...) range_cond(keyX) := condition that allows to construct range access of keyX and doesn't allow to construct range/index_merge accesses for any keys of the table in question. For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: SEL_TREE( range(key1_1), ... range(key2_1), SEL_IMERGE( (1) SEL_TREE(key3_1}) SEL_TREE(key3_2}) ... ) ... ) which can be used to make a cost-based choice between range and index_merge. Limitations ----------- This will not be a full solution in a sense that the range analyzer will not be able to produce sel_tree (1) if the WHERE clause is specified in other form (e.g. brackets were opened). TODO ---- * is it a problem if there are keys that are referred to both from index_merge and from range access? * How strict is the limitation on the form of the WHERE? * Which version should this be based on? 5.1? Which patches are should be in (google's/percona's/maria/etc?) * TODO: The optimizer didn't compare costs of index_merge and range before (ok it did but that was done for accesses to different tables). Will there be any possible gotchas here? LOW-LEVEL DESIGN: <contents> 1. Current implementation overview 1.1. Problems in the current implementation 2. New implementation 2.1 New tree_and() 2.2 New tree_or() 3. Testing and required coverage </contents> 1. Current implementation overview ================================== At the moment, range analyzer works as follows: SEL_TREE structure represents # There are sel_trees, a sel_tree is either range or merge tree sel_tree = range_tree | imerge_tree # a range tree has range access options, possibly for several keys range_tree = range(key1) AND range(key2) AND ... AND range(keyN); (here range(keyi) may represent ranges not for initial keyi prefixes, but ranges for any infixes for keyi) # merge tree represents several way to index_merge imerge_tree = imerge1 AND imerge2 AND ... # a way to do index merge == a set to use of different indexes. imergeX = range_tree1 OR range_tree2 OR .. where no pair of range_treeX have ranges over the same index. tree_and(A, B) { if (both A and B are range trees) return a range_tree with computed intersection for each range; if (only one of A and B is a range tree) return that tree; // DISCARD-IMERGE-1 // at this point both trees are index_merge trees return concat_lists( A.imerge1 ... A.imergeN, B.imerge1 ... B.imergeN); } tree_or(A, B) { if (A and B are range trees) { R = new range_tree; for each index i R.add(range_union(A.range(i), B.range(i))); if (R has at least one range access) return R; // DISCARD-IMERGE-2 else { /* could not build any range accesses. construct index_merge */ remove non-ranges from A; remove non-ranges from B; return new index_merge(A, B); // DISCARD-IMERGE-3 } } else if (A is range tree and B is index_merge tree (or vice versa)) { Perform this transformation: range_treeA // this is A OR (range_treeB_11 OR range_treeB_12 OR ... OR range_treeB_1N) AND (range_treeB_21 OR range_treeB_22 OR ... OR range_treeB_2N) AND ... (range_treeB_K1 OR range_treeB_K2 OR ... OR range_treeB_kN) = (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND (range_treeA OR range_treeB_21 OR ... OR range_treeB_2N) AND ... (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) Now each line represents an index_merge.. } else if (both A and B are index_merge trees) { Perform this transformation: imergeA1 AND imergeA2 AND ... AND imergeAN OR imergeB1 AND imergeB2 AND ... AND imergeBN -> (discard all imergeA{i=2,3,...} -> // DISCARD-IMERGE-4 imergeA1 OR imergeB1 = = (combine imergeA1 with each of the range_treeB_1{i} ) = combine(imergeA1 OR range_treeB_11) AND combine(imergeA1 OR range_treeB_12) AND ... AND combine(imergeA1 OR range_treeB_1N) } } 1.1. Problems in the current implementation ------------------------------------------- As marked in the code above: DISCARD-IMERGE-1 step will cause index_merge option to be discarded when the WHERE clause has this form: (t.key1=c1 OR t.key2=c2) AND t.badkey < c3 DISCARD-IMERGE-2 step will cause index_merge option to be discarded when the WHERE clause has this form (conditions t.badkey may have abritrary form): (t.badkey<c1 AND t.key1=c1) OR (t.key2=c2 AND t.badkey < c2) DISCARD-IMERGE-3 manifests itself as the following effect: suppose there are two indexes: INDEX i1(col1, col2), INDEX i2(col1, col3) and this WHERE clause: col1=c1 AND (col2=c2 OR col3=c3) The optimizer will generate the plans that only use the "col1=c1" part. The right side of the AND will be ignored even if it has good selectivity. (Here no imerge for col2=c2 OR col3=c3 will be built since neither col2=c2 nor col3=c3 represent index ranges.) 2. New implementation ===================== <general idea> * Don't start fighting combinatorial explosion until we've actually got one. </> SEL_TREE structure will be now able to hold both index_merge and range scan candidates at the same time. That is, sel_tree2 = range_tree AND imerge_tree where both parts are optional (i.e. can be empty) Operations on SEL_ARG trees will be modified to produce/process the trees of this kind: 2.1 New tree_and() ------------------ In order not to lose plans, we'll make these changes: A1. Don't remove index_merge part of the tree (this will take care of DISCARD-IMERGE-1 problem) A2. Push range conditions down into index_merge trees that may support them. if one tree has range(key1) and the other tree has imerge(key1 OR key2) then perform an equvalent of this operation: rangeA(key1) AND ( rangeB(key1) OR rangeB(key2)) = (rangeA(key1) AND rangeB(key1)) OR (rangeA(key1) AND rangeB(key2)) A3. Just as before: if both sel_tree A and sel_tree B have index_merge options, concatenate them together. 2.2 New tree_or() ----------------- O1. Dont remove non-range plans: Current tree_or() code will refuse to produce index_merge plans for conditions like "t.key1part2=const OR t.key2part1=const" (this is marked as DISCARD-IMERGE-3). This was justifed as the left part of the AND condition is not usable for range access, and the operation of tree_and() guaranteed that there was no way it could changed to make a usable range plan. With new tree_and() and rule A2, this is no longer the case. For example for this query: (t.key1part2=const OR t.key2part1=const) AND t.key1part1=const it will construct a imerge(t.key1part2=const OR t.key2part1=const), range(t.key1part1=const) then tree_and() will apply rule A2 to push the range down into index merge and after that we'll have: range(t.key1part1=const) imerge( t.key1part2=const AND t.key1part1=const, t.key2part1=const ) note that imerge(...) describes a usable index_merge plan and it's possible that it will be the best access path. O2. "Create index_merge accesses when possible" Current tree_or() will not create index_merge access when it could create non-index merge access (see DISCARD-IMERGE-2 and its example in the "Problems in the current implementation" section). This will be changed to work as follows: we will create index_merge made for index scans that didn't have their match in the other sel_tree. Ilustrating it with an example: | sel_tree_A | sel_tree_B | A or B | include in index_merge? ------+------------+------------+--------+------------------------ key1 | cond1 | cond2 | condM | no key2 | cond3 | cond4 | NULL | no key3 | cond5 | | | yes, A-side key4 | cond6 | | | yes, A-side key5 | | cond7 | | yes, B-side key6 | | cond8 | | yes, B-side here we assume that - (cond1 OR cond2) did produce a combined range. Not including them in index_merge. - (cond3 OR cond4) didn't produce a usable range (e.g. they were t.key1part1=c1 AND t.key1part2=c1, respectively, and combining them didn't yield any range list) - All other scand didn't have their counterparts, so we'll end up with a SEL_TREE of: range(condM) AND index_merge((cond5 AND cond6),(cond7 AND cond8)) . O4. There is no O4. DISCARD-INDEX-MERGE-4 will remain there. The idea is that although DISCARD-INDEX-MERGE-4 does discard plans, so far we haven seen any complaints that could be attributed to it. If we face the need to lift DISCARD-INDEX-MERGE-4, our answer will be to lift it ,and produce a cross-product: ((key1p OR key2p) AND (key3p OR key4p)) OR ((key5p OR key6p) AND (key7p OR key8p)) = (key1p OR key2p OR key5p OR key6p) AND // this part is currently (key3p OR key4p OR key5p OR key6p) AND // produced (key1p OR key2p OR key5p OR key6p) AND // this part will be added (key3p OR key4p OR key5p OR key6p) //. In order to limit the impact of this combinatorial explosion, we will introduce a rule that we won't generate more than #defined MAX_IMERGE_OPTS options. 3. Testing and required coverage ================================ So far could find the following user cases: * BUG#17259: Query optimizer chooses wrong index * BUG#17673: Optimizer does not use Index Merge optimization in some cases * BUG#23322: Optimizer sometimes erroniously prefers other index over index merge * BUG#30151: optimizer is very reluctant to chose index_merge algorithm ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): index_merge: fair choice between index_merge union and range access (24)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: index_merge: fair choice between index_merge union and range access CREATION DATE..: Tue, 26 May 2009, 12:10 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: Psergey CATEGORY.......: Server-Sprint TASK ID........: 24 (http://askmonty.org/worklog/?tid=24) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Sun, 16 Aug 2009, 02:13)=-=- Low Level Design modified. --- /tmp/wklog.24.old.23383 2009-08-16 02:13:54.000000000 +0300 +++ /tmp/wklog.24.new.23383 2009-08-16 02:13:54.000000000 +0300 @@ -125,7 +125,7 @@ The optimizer will generate the plans that only use the "col1=c1" part. The right side of the AND will be ignored even if it has good selectivity. -(Here an imerge for col2=c2 OR col3=c3 won't be built since neither col2=c2 nor +(Here no imerge for col2=c2 OR col3=c3 will be built since neither col2=c2 nor col3=c3 represent index ranges.) @@ -199,7 +199,7 @@ O2. "Create index_merge accesses when possible" Current tree_or() will not create index_merge access when it could create - non-index merge access (see DISCARD-IMERGE-3 and its example in the "Problems + non-index merge access (see DISCARD-IMERGE-2 and its example in the "Problems in the current implementation" section). This will be changed to work as follows: we will create index_merge made for index scans that didn't have their match in the other sel_tree. -=-=(Guest - Sun, 16 Aug 2009, 01:03)=-=- Low Level Design modified. --- /tmp/wklog.24.old.20767 2009-08-16 01:03:11.000000000 +0300 +++ /tmp/wklog.24.new.20767 2009-08-16 01:03:11.000000000 +0300 @@ -18,6 +18,8 @@ # a range tree has range access options, possibly for several keys range_tree = range(key1) AND range(key2) AND ... AND range(keyN); + (here range(keyi) may represent ranges not for initial keyi prefixes, + but ranges for any infixes for keyi) # merge tree represents several way to index_merge imerge_tree = imerge1 AND imerge2 AND ... @@ -47,13 +49,13 @@ R.add(range_union(A.range(i), B.range(i))); if (R has at least one range access) - return R; + return R; // DISCARD-IMERGE-2 else { /* could not build any range accesses. construct index_merge */ - remove non-ranges from A; // DISCARD-IMERGE-2 + remove non-ranges from A; remove non-ranges from B; - return new index_merge(A, B); + return new index_merge(A, B); // DISCARD-IMERGE-3 } } else if (A is range tree and B is index_merge tree (or vice versa)) @@ -65,12 +67,12 @@ (range_treeB_11 OR range_treeB_12 OR ... OR range_treeB_1N) AND (range_treeB_21 OR range_treeB_22 OR ... OR range_treeB_2N) AND ... - (range_treeB_K1 OR range_treeB_K2 OR ... OR range_treeB_kN) AND + (range_treeB_K1 OR range_treeB_K2 OR ... OR range_treeB_kN) = (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND (range_treeA OR range_treeB_21 OR ... OR range_treeB_2N) AND ... - (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND + (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) Now each line represents an index_merge.. } @@ -82,18 +84,18 @@ OR imergeB1 AND imergeB2 AND ... AND imergeBN - -> (discard all imergeA{i=2,3,...} -> // DISCARD-IMERGE-3 + -> (discard all imergeA{i=2,3,...} -> // DISCARD-IMERGE-4 imergeA1 OR - imergeB1 AND imergeB2 AND ... AND imergeBN = + imergeB1 = - = (combine imergeA1 with each of the imergeB{i} ) = + = (combine imergeA1 with each of the range_treeB_1{i} ) = - combine(imergeA1 OR imergeB1) AND - combine(imergeA1 OR imergeB2) AND + combine(imergeA1 OR range_treeB_11) AND + combine(imergeA1 OR range_treeB_12) AND ... AND - combine(imergeA1 OR imergeBN) + combine(imergeA1 OR range_treeB_1N) } } @@ -109,7 +111,7 @@ DISCARD-IMERGE-2 step will cause index_merge option to be discarded when the WHERE clause has this form (conditions t.badkey may have abritrary form): - (t.badkey<c1 AND t.key1=c1) OR (t.key1=c2 AND t.badkey < c2) + (t.badkey<c1 AND t.key1=c1) OR (t.key2=c2 AND t.badkey < c2) DISCARD-IMERGE-3 manifests itself as the following effect: suppose there are two indexes: @@ -123,6 +125,8 @@ The optimizer will generate the plans that only use the "col1=c1" part. The right side of the AND will be ignored even if it has good selectivity. +(Here an imerge for col2=c2 OR col3=c3 won't be built since neither col2=c2 nor +col3=c3 represent index ranges.) 2. New implementation -=-=(Guest - Mon, 20 Jul 2009, 17:13)=-=- Dependency deleted: 30 no longer depends on 24 -=-=(Guest - Sat, 20 Jun 2009, 09:34)=-=- Low Level Design modified. --- /tmp/wklog.24.old.21663 2009-06-20 09:34:48.000000000 +0300 +++ /tmp/wklog.24.new.21663 2009-06-20 09:34:48.000000000 +0300 @@ -4,6 +4,7 @@ 2. New implementation 2.1 New tree_and() 2.2 New tree_or() +3. Testing and required coverage </contents> 1. Current implementation overview @@ -240,3 +241,14 @@ In order to limit the impact of this combinatorial explosion, we will introduce a rule that we won't generate more than #defined MAX_IMERGE_OPTS options. + +3. Testing and required coverage +================================ +So far could find the following user cases: + +* BUG#17259: Query optimizer chooses wrong index +* BUG#17673: Optimizer does not use Index Merge optimization in some cases +* BUG#23322: Optimizer sometimes erroniously prefers other index over index merge +* BUG#30151: optimizer is very reluctant to chose index_merge algorithm + + -=-=(Guest - Thu, 18 Jun 2009, 16:55)=-=- Low Level Design modified. --- /tmp/wklog.24.old.19152 2009-06-18 16:55:00.000000000 +0300 +++ /tmp/wklog.24.new.19152 2009-06-18 16:55:00.000000000 +0300 @@ -141,13 +141,15 @@ Operations on SEL_ARG trees will be modified to produce/process the trees of this kind: + 2.1 New tree_and() ------------------ In order not to lose plans, we'll make these changes: -1. Don't remove index_merge part of the tree. +A1. Don't remove index_merge part of the tree (this will take care of + DISCARD-IMERGE-1 problem) -2. Push range conditions down into index_merge trees that may support them. +A2. Push range conditions down into index_merge trees that may support them. if one tree has range(key1) and the other tree has imerge(key1 OR key2) then perform an equvalent of this operation: @@ -155,8 +157,86 @@ (rangeA(key1) AND rangeB(key1)) OR (rangeA(key1) AND rangeB(key2)) -3. Just as before: if both sel_tree A and sel_tree B have index_merge options, +A3. Just as before: if both sel_tree A and sel_tree B have index_merge options, concatenate them together. -2.2 New tree_or() +2.2 New tree_or() +----------------- +O1. Dont remove non-range plans: + Current tree_or() code will refuse to produce index_merge plans for + conditions like + + "t.key1part2=const OR t.key2part1=const" + + (this is marked as DISCARD-IMERGE-3). This was justifed as the left part of + the AND condition is not usable for range access, and the operation of + tree_and() guaranteed that there was no way it could changed to make a + usable range plan. With new tree_and() and rule A2, this is no longer the + case. For example for this query: + + (t.key1part2=const OR t.key2part1=const) AND t.key1part1=const + + it will construct a + + imerge(t.key1part2=const OR t.key2part1=const), range(t.key1part1=const) + + then tree_and() will apply rule A2 to push the range down into index merge + and after that we'll have: + + range(t.key1part1=const) + imerge( + t.key1part2=const AND t.key1part1=const, + t.key2part1=const + ) + note that imerge(...) describes a usable index_merge plan and it's possible + that it will be the best access path. + +O2. "Create index_merge accesses when possible" + Current tree_or() will not create index_merge access when it could create + non-index merge access (see DISCARD-IMERGE-3 and its example in the "Problems + in the current implementation" section). This will be changed to work as + follows: we will create index_merge made for index scans that didn't have + their match in the other sel_tree. + Ilustrating it with an example: + + | sel_tree_A | sel_tree_B | A or B | include in index_merge? + ------+------------+------------+--------+------------------------ + key1 | cond1 | cond2 | condM | no + key2 | cond3 | cond4 | NULL | no + key3 | cond5 | | | yes, A-side + key4 | cond6 | | | yes, A-side + key5 | | cond7 | | yes, B-side + key6 | | cond8 | | yes, B-side + + here we assume that + - (cond1 OR cond2) did produce a combined range. Not including them in + index_merge. + - (cond3 OR cond4) didn't produce a usable range (e.g. they were + t.key1part1=c1 AND t.key1part2=c1, respectively, and combining them + didn't yield any range list) + - All other scand didn't have their counterparts, so we'll end up with a + SEL_TREE of: + + range(condM) AND index_merge((cond5 AND cond6),(cond7 AND cond8)) + . + +O4. There is no O4. DISCARD-INDEX-MERGE-4 will remain there. The idea is +that although DISCARD-INDEX-MERGE-4 does discard plans, so far we haven +seen any complaints that could be attributed to it. +If we face the need to lift DISCARD-INDEX-MERGE-4, our answer will be to +lift it ,and produce a cross-product: + + ((key1p OR key2p) AND (key3p OR key4p)) + OR + ((key5p OR key6p) AND (key7p OR key8p)) + + = (key1p OR key2p OR key5p OR key6p) AND // this part is currently + (key3p OR key4p OR key5p OR key6p) AND // produced + + (key1p OR key2p OR key5p OR key6p) AND // this part will be added + (key3p OR key4p OR key5p OR key6p) //. + +In order to limit the impact of this combinatorial explosion, we will +introduce a rule that we won't generate more than #defined +MAX_IMERGE_OPTS options. -=-=(Guest - Thu, 18 Jun 2009, 14:56)=-=- Low Level Design modified. --- /tmp/wklog.24.old.15612 2009-06-18 14:56:09.000000000 +0300 +++ /tmp/wklog.24.new.15612 2009-06-18 14:56:09.000000000 +0300 @@ -1 +1,162 @@ +<contents> +1. Current implementation overview +1.1. Problems in the current implementation +2. New implementation +2.1 New tree_and() +2.2 New tree_or() +</contents> + +1. Current implementation overview +================================== +At the moment, range analyzer works as follows: + +SEL_TREE structure represents + + # There are sel_trees, a sel_tree is either range or merge tree + sel_tree = range_tree | imerge_tree + + # a range tree has range access options, possibly for several keys + range_tree = range(key1) AND range(key2) AND ... AND range(keyN); + + # merge tree represents several way to index_merge + imerge_tree = imerge1 AND imerge2 AND ... + + # a way to do index merge == a set to use of different indexes. + imergeX = range_tree1 OR range_tree2 OR .. + where no pair of range_treeX have ranges over the same index. + + + tree_and(A, B) + { + if (both A and B are range trees) + return a range_tree with computed intersection for each range; + if (only one of A and B is a range tree) + return that tree; // DISCARD-IMERGE-1 + // at this point both trees are index_merge trees + return concat_lists( A.imerge1 ... A.imergeN, B.imerge1 ... B.imergeN); + } + + + tree_or(A, B) + { + if (A and B are range trees) + { + R = new range_tree; + for each index i + R.add(range_union(A.range(i), B.range(i))); + + if (R has at least one range access) + return R; + else + { + /* could not build any range accesses. construct index_merge */ + remove non-ranges from A; // DISCARD-IMERGE-2 + remove non-ranges from B; + return new index_merge(A, B); + } + } + else if (A is range tree and B is index_merge tree (or vice versa)) + { + Perform this transformation: + + range_treeA // this is A + OR + (range_treeB_11 OR range_treeB_12 OR ... OR range_treeB_1N) AND + (range_treeB_21 OR range_treeB_22 OR ... OR range_treeB_2N) AND + ... + (range_treeB_K1 OR range_treeB_K2 OR ... OR range_treeB_kN) AND + = + (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND + (range_treeA OR range_treeB_21 OR ... OR range_treeB_2N) AND + ... + (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND + + Now each line represents an index_merge.. + } + else if (both A and B are index_merge trees) + { + Perform this transformation: + + imergeA1 AND imergeA2 AND ... AND imergeAN + OR + imergeB1 AND imergeB2 AND ... AND imergeBN + + -> (discard all imergeA{i=2,3,...} -> // DISCARD-IMERGE-3 + + imergeA1 + OR + imergeB1 AND imergeB2 AND ... AND imergeBN = + + = (combine imergeA1 with each of the imergeB{i} ) = + + combine(imergeA1 OR imergeB1) AND + combine(imergeA1 OR imergeB2) AND + ... AND + combine(imergeA1 OR imergeBN) + } + } + +1.1. Problems in the current implementation +------------------------------------------- +As marked in the code above: + +DISCARD-IMERGE-1 step will cause index_merge option to be discarded when +the WHERE clause has this form: + + (t.key1=c1 OR t.key2=c2) AND t.badkey < c3 + +DISCARD-IMERGE-2 step will cause index_merge option to be discarded when +the WHERE clause has this form (conditions t.badkey may have abritrary form): + + (t.badkey<c1 AND t.key1=c1) OR (t.key1=c2 AND t.badkey < c2) + +DISCARD-IMERGE-3 manifests itself as the following effect: suppose there are +two indexes: + + INDEX i1(col1, col2), + INDEX i2(col1, col3) + +and this WHERE clause: + + col1=c1 AND (col2=c2 OR col3=c3) + +The optimizer will generate the plans that only use the "col1=c1" part. The +right side of the AND will be ignored even if it has good selectivity. + + +2. New implementation +===================== + +<general idea> +* Don't start fighting combinatorial explosion until we've actually got one. +</> + +SEL_TREE structure will be now able to hold both index_merge and range scan +candidates at the same time. That is, + + sel_tree2 = range_tree AND imerge_tree + +where both parts are optional (i.e. can be empty) + +Operations on SEL_ARG trees will be modified to produce/process the trees of +this kind: + +2.1 New tree_and() +------------------ +In order not to lose plans, we'll make these changes: + +1. Don't remove index_merge part of the tree. + +2. Push range conditions down into index_merge trees that may support them. + if one tree has range(key1) and the other tree has imerge(key1 OR key2) + then perform an equvalent of this operation: + + rangeA(key1) AND ( rangeB(key1) OR rangeB(key2)) = + + (rangeA(key1) AND rangeB(key1)) OR (rangeA(key1) AND rangeB(key2)) + +3. Just as before: if both sel_tree A and sel_tree B have index_merge options, + concatenate them together. + +2.2 New tree_or() -=-=(Psergey - Wed, 03 Jun 2009, 12:09)=-=- Dependency created: 30 now depends on 24 -=-=(Guest - Mon, 01 Jun 2009, 23:30)=-=- High-Level Specification modified. --- /tmp/wklog.24.old.21580 2009-06-01 23:30:06.000000000 +0300 +++ /tmp/wklog.24.new.21580 2009-06-01 23:30:06.000000000 +0300 @@ -64,6 +64,9 @@ * How strict is the limitation on the form of the WHERE? +* Which version should this be based on? 5.1? Which patches are should be in + (google's/percona's/maria/etc?) + * TODO: The optimizer didn't compare costs of index_merge and range before (ok it did but that was done for accesses to different tables). Will there be any possible gotchas here? -=-=(Guest - Wed, 27 May 2009, 13:59)=-=- Title modified. --- /tmp/wklog.24.old.9498 2009-05-27 13:59:23.000000000 +0300 +++ /tmp/wklog.24.new.9498 2009-05-27 13:59:23.000000000 +0300 @@ -1 +1 @@ -index_merge optimizer: dont discard index_merge union strategies when range is available +index_merge: fair choice between index_merge union and range access -=-=(Guest - Tue, 26 May 2009, 13:27)=-=- High-Level Specification modified. --- /tmp/wklog.24.old.305 2009-05-26 13:27:32.000000000 +0300 +++ /tmp/wklog.24.new.305 2009-05-26 13:27:32.000000000 +0300 @@ -1 +1,70 @@ +(Not a ready HLS but draft) +<contents> +Solution overview +Limitations +TODO + +</contents> + +Solution overview +================= +The idea is to delay discarding potential index_merge plans until the point +where it is really necessary. + +This way, we won't have to do much changes in the range analyzer, but will be +able to keep potential index_merge plan just enough so that it's possible to +take it into consideration together with range access plans. + +Since there are no changes in the optimizer, the ability to consider both +range and index_merge options will be limited to WHERE clauses of this form: + + WHERE := range_cond(key1_1) AND + range_cond(key2_1) AND + other_cond AND + index_merge_OR_cond1(key3_1, key3_2, ...) + index_merge_OR_cond2(key4_1, key4_2, ...) + +where + + index_merge_OR_cond{N} := (range_cond(keyN_1) OR + range_cond(keyN_2) OR ...) + + + range_cond(keyX) := condition that allows to construct range access of keyX + and doesn't allow to construct range/index_merge accesses + for any keys of the table in question. + + +For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: + + SEL_TREE( + range(key1_1), + ... + range(key2_1), + SEL_IMERGE( (1) + SEL_TREE(key3_1}) + SEL_TREE(key3_2}) + ... + ) + ... + ) + +which can be used to make a cost-based choice between range and index_merge. + +Limitations +----------- +This will not be a full solution in a sense that the range analyzer will not +be able to produce sel_tree (1) if the WHERE clause is specified in other form +(e.g. brackets were opened). + +TODO +---- +* is it a problem if there are keys that are referred to both from + index_merge and from range access? + +* How strict is the limitation on the form of the WHERE? + +* TODO: The optimizer didn't compare costs of index_merge and range before (ok + it did but that was done for accesses to different tables). Will there be any + possible gotchas here? DESCRIPTION: Current range optimizer will discard possible index_merge/[sort]union strategies when there is a possible range plan. This action is a part of measures we take to avoid combinatorial explosion of possible range/ index_merge strategies. A bad side effect of this is that for WHERE clauses in form t.key1= 'very-frequent-value' AND (t.key2='rare-value1' OR t.key3='rare-value2') the optimizer will - discard union(key2,key3) in favor of range(key1) - consider costs of using range(key1) and discard that plan also and the overall effect is that possible poor range access will cause possible good index_merge access not to be considered. This WL is to about lifting this limitation at least for some subset of WHERE clauses. HIGH-LEVEL SPECIFICATION: (Not a ready HLS but draft) <contents> Solution overview Limitations TODO </contents> Solution overview ================= The idea is to delay discarding potential index_merge plans until the point where it is really necessary. This way, we won't have to do much changes in the range analyzer, but will be able to keep potential index_merge plan just enough so that it's possible to take it into consideration together with range access plans. Since there are no changes in the optimizer, the ability to consider both range and index_merge options will be limited to WHERE clauses of this form: WHERE := range_cond(key1_1) AND range_cond(key2_1) AND other_cond AND index_merge_OR_cond1(key3_1, key3_2, ...) index_merge_OR_cond2(key4_1, key4_2, ...) where index_merge_OR_cond{N} := (range_cond(keyN_1) OR range_cond(keyN_2) OR ...) range_cond(keyX) := condition that allows to construct range access of keyX and doesn't allow to construct range/index_merge accesses for any keys of the table in question. For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: SEL_TREE( range(key1_1), ... range(key2_1), SEL_IMERGE( (1) SEL_TREE(key3_1}) SEL_TREE(key3_2}) ... ) ... ) which can be used to make a cost-based choice between range and index_merge. Limitations ----------- This will not be a full solution in a sense that the range analyzer will not be able to produce sel_tree (1) if the WHERE clause is specified in other form (e.g. brackets were opened). TODO ---- * is it a problem if there are keys that are referred to both from index_merge and from range access? * How strict is the limitation on the form of the WHERE? * Which version should this be based on? 5.1? Which patches are should be in (google's/percona's/maria/etc?) * TODO: The optimizer didn't compare costs of index_merge and range before (ok it did but that was done for accesses to different tables). Will there be any possible gotchas here? LOW-LEVEL DESIGN: <contents> 1. Current implementation overview 1.1. Problems in the current implementation 2. New implementation 2.1 New tree_and() 2.2 New tree_or() 3. Testing and required coverage </contents> 1. Current implementation overview ================================== At the moment, range analyzer works as follows: SEL_TREE structure represents # There are sel_trees, a sel_tree is either range or merge tree sel_tree = range_tree | imerge_tree # a range tree has range access options, possibly for several keys range_tree = range(key1) AND range(key2) AND ... AND range(keyN); (here range(keyi) may represent ranges not for initial keyi prefixes, but ranges for any infixes for keyi) # merge tree represents several way to index_merge imerge_tree = imerge1 AND imerge2 AND ... # a way to do index merge == a set to use of different indexes. imergeX = range_tree1 OR range_tree2 OR .. where no pair of range_treeX have ranges over the same index. tree_and(A, B) { if (both A and B are range trees) return a range_tree with computed intersection for each range; if (only one of A and B is a range tree) return that tree; // DISCARD-IMERGE-1 // at this point both trees are index_merge trees return concat_lists( A.imerge1 ... A.imergeN, B.imerge1 ... B.imergeN); } tree_or(A, B) { if (A and B are range trees) { R = new range_tree; for each index i R.add(range_union(A.range(i), B.range(i))); if (R has at least one range access) return R; // DISCARD-IMERGE-2 else { /* could not build any range accesses. construct index_merge */ remove non-ranges from A; remove non-ranges from B; return new index_merge(A, B); // DISCARD-IMERGE-3 } } else if (A is range tree and B is index_merge tree (or vice versa)) { Perform this transformation: range_treeA // this is A OR (range_treeB_11 OR range_treeB_12 OR ... OR range_treeB_1N) AND (range_treeB_21 OR range_treeB_22 OR ... OR range_treeB_2N) AND ... (range_treeB_K1 OR range_treeB_K2 OR ... OR range_treeB_kN) = (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND (range_treeA OR range_treeB_21 OR ... OR range_treeB_2N) AND ... (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) Now each line represents an index_merge.. } else if (both A and B are index_merge trees) { Perform this transformation: imergeA1 AND imergeA2 AND ... AND imergeAN OR imergeB1 AND imergeB2 AND ... AND imergeBN -> (discard all imergeA{i=2,3,...} -> // DISCARD-IMERGE-4 imergeA1 OR imergeB1 = = (combine imergeA1 with each of the range_treeB_1{i} ) = combine(imergeA1 OR range_treeB_11) AND combine(imergeA1 OR range_treeB_12) AND ... AND combine(imergeA1 OR range_treeB_1N) } } 1.1. Problems in the current implementation ------------------------------------------- As marked in the code above: DISCARD-IMERGE-1 step will cause index_merge option to be discarded when the WHERE clause has this form: (t.key1=c1 OR t.key2=c2) AND t.badkey < c3 DISCARD-IMERGE-2 step will cause index_merge option to be discarded when the WHERE clause has this form (conditions t.badkey may have abritrary form): (t.badkey<c1 AND t.key1=c1) OR (t.key2=c2 AND t.badkey < c2) DISCARD-IMERGE-3 manifests itself as the following effect: suppose there are two indexes: INDEX i1(col1, col2), INDEX i2(col1, col3) and this WHERE clause: col1=c1 AND (col2=c2 OR col3=c3) The optimizer will generate the plans that only use the "col1=c1" part. The right side of the AND will be ignored even if it has good selectivity. (Here no imerge for col2=c2 OR col3=c3 will be built since neither col2=c2 nor col3=c3 represent index ranges.) 2. New implementation ===================== <general idea> * Don't start fighting combinatorial explosion until we've actually got one. </> SEL_TREE structure will be now able to hold both index_merge and range scan candidates at the same time. That is, sel_tree2 = range_tree AND imerge_tree where both parts are optional (i.e. can be empty) Operations on SEL_ARG trees will be modified to produce/process the trees of this kind: 2.1 New tree_and() ------------------ In order not to lose plans, we'll make these changes: A1. Don't remove index_merge part of the tree (this will take care of DISCARD-IMERGE-1 problem) A2. Push range conditions down into index_merge trees that may support them. if one tree has range(key1) and the other tree has imerge(key1 OR key2) then perform an equvalent of this operation: rangeA(key1) AND ( rangeB(key1) OR rangeB(key2)) = (rangeA(key1) AND rangeB(key1)) OR (rangeA(key1) AND rangeB(key2)) A3. Just as before: if both sel_tree A and sel_tree B have index_merge options, concatenate them together. 2.2 New tree_or() ----------------- O1. Dont remove non-range plans: Current tree_or() code will refuse to produce index_merge plans for conditions like "t.key1part2=const OR t.key2part1=const" (this is marked as DISCARD-IMERGE-3). This was justifed as the left part of the AND condition is not usable for range access, and the operation of tree_and() guaranteed that there was no way it could changed to make a usable range plan. With new tree_and() and rule A2, this is no longer the case. For example for this query: (t.key1part2=const OR t.key2part1=const) AND t.key1part1=const it will construct a imerge(t.key1part2=const OR t.key2part1=const), range(t.key1part1=const) then tree_and() will apply rule A2 to push the range down into index merge and after that we'll have: range(t.key1part1=const) imerge( t.key1part2=const AND t.key1part1=const, t.key2part1=const ) note that imerge(...) describes a usable index_merge plan and it's possible that it will be the best access path. O2. "Create index_merge accesses when possible" Current tree_or() will not create index_merge access when it could create non-index merge access (see DISCARD-IMERGE-2 and its example in the "Problems in the current implementation" section). This will be changed to work as follows: we will create index_merge made for index scans that didn't have their match in the other sel_tree. Ilustrating it with an example: | sel_tree_A | sel_tree_B | A or B | include in index_merge? ------+------------+------------+--------+------------------------ key1 | cond1 | cond2 | condM | no key2 | cond3 | cond4 | NULL | no key3 | cond5 | | | yes, A-side key4 | cond6 | | | yes, A-side key5 | | cond7 | | yes, B-side key6 | | cond8 | | yes, B-side here we assume that - (cond1 OR cond2) did produce a combined range. Not including them in index_merge. - (cond3 OR cond4) didn't produce a usable range (e.g. they were t.key1part1=c1 AND t.key1part2=c1, respectively, and combining them didn't yield any range list) - All other scand didn't have their counterparts, so we'll end up with a SEL_TREE of: range(condM) AND index_merge((cond5 AND cond6),(cond7 AND cond8)) . O4. There is no O4. DISCARD-INDEX-MERGE-4 will remain there. The idea is that although DISCARD-INDEX-MERGE-4 does discard plans, so far we haven seen any complaints that could be attributed to it. If we face the need to lift DISCARD-INDEX-MERGE-4, our answer will be to lift it ,and produce a cross-product: ((key1p OR key2p) AND (key3p OR key4p)) OR ((key5p OR key6p) AND (key7p OR key8p)) = (key1p OR key2p OR key5p OR key6p) AND // this part is currently (key3p OR key4p OR key5p OR key6p) AND // produced (key1p OR key2p OR key5p OR key6p) AND // this part will be added (key3p OR key4p OR key5p OR key6p) //. In order to limit the impact of this combinatorial explosion, we will introduce a rule that we won't generate more than #defined MAX_IMERGE_OPTS options. 3. Testing and required coverage ================================ So far could find the following user cases: * BUG#17259: Query optimizer chooses wrong index * BUG#17673: Optimizer does not use Index Merge optimization in some cases * BUG#23322: Optimizer sometimes erroniously prefers other index over index merge * BUG#30151: optimizer is very reluctant to chose index_merge algorithm ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): index_merge: fair choice between index_merge union and range access (24)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: index_merge: fair choice between index_merge union and range access CREATION DATE..: Tue, 26 May 2009, 12:10 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: Psergey CATEGORY.......: Server-Sprint TASK ID........: 24 (http://askmonty.org/worklog/?tid=24) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Sun, 16 Aug 2009, 01:03)=-=- Low Level Design modified. --- /tmp/wklog.24.old.20767 2009-08-16 01:03:11.000000000 +0300 +++ /tmp/wklog.24.new.20767 2009-08-16 01:03:11.000000000 +0300 @@ -18,6 +18,8 @@ # a range tree has range access options, possibly for several keys range_tree = range(key1) AND range(key2) AND ... AND range(keyN); + (here range(keyi) may represent ranges not for initial keyi prefixes, + but ranges for any infixes for keyi) # merge tree represents several way to index_merge imerge_tree = imerge1 AND imerge2 AND ... @@ -47,13 +49,13 @@ R.add(range_union(A.range(i), B.range(i))); if (R has at least one range access) - return R; + return R; // DISCARD-IMERGE-2 else { /* could not build any range accesses. construct index_merge */ - remove non-ranges from A; // DISCARD-IMERGE-2 + remove non-ranges from A; remove non-ranges from B; - return new index_merge(A, B); + return new index_merge(A, B); // DISCARD-IMERGE-3 } } else if (A is range tree and B is index_merge tree (or vice versa)) @@ -65,12 +67,12 @@ (range_treeB_11 OR range_treeB_12 OR ... OR range_treeB_1N) AND (range_treeB_21 OR range_treeB_22 OR ... OR range_treeB_2N) AND ... - (range_treeB_K1 OR range_treeB_K2 OR ... OR range_treeB_kN) AND + (range_treeB_K1 OR range_treeB_K2 OR ... OR range_treeB_kN) = (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND (range_treeA OR range_treeB_21 OR ... OR range_treeB_2N) AND ... - (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND + (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) Now each line represents an index_merge.. } @@ -82,18 +84,18 @@ OR imergeB1 AND imergeB2 AND ... AND imergeBN - -> (discard all imergeA{i=2,3,...} -> // DISCARD-IMERGE-3 + -> (discard all imergeA{i=2,3,...} -> // DISCARD-IMERGE-4 imergeA1 OR - imergeB1 AND imergeB2 AND ... AND imergeBN = + imergeB1 = - = (combine imergeA1 with each of the imergeB{i} ) = + = (combine imergeA1 with each of the range_treeB_1{i} ) = - combine(imergeA1 OR imergeB1) AND - combine(imergeA1 OR imergeB2) AND + combine(imergeA1 OR range_treeB_11) AND + combine(imergeA1 OR range_treeB_12) AND ... AND - combine(imergeA1 OR imergeBN) + combine(imergeA1 OR range_treeB_1N) } } @@ -109,7 +111,7 @@ DISCARD-IMERGE-2 step will cause index_merge option to be discarded when the WHERE clause has this form (conditions t.badkey may have abritrary form): - (t.badkey<c1 AND t.key1=c1) OR (t.key1=c2 AND t.badkey < c2) + (t.badkey<c1 AND t.key1=c1) OR (t.key2=c2 AND t.badkey < c2) DISCARD-IMERGE-3 manifests itself as the following effect: suppose there are two indexes: @@ -123,6 +125,8 @@ The optimizer will generate the plans that only use the "col1=c1" part. The right side of the AND will be ignored even if it has good selectivity. +(Here an imerge for col2=c2 OR col3=c3 won't be built since neither col2=c2 nor +col3=c3 represent index ranges.) 2. New implementation -=-=(Guest - Mon, 20 Jul 2009, 17:13)=-=- Dependency deleted: 30 no longer depends on 24 -=-=(Guest - Sat, 20 Jun 2009, 09:34)=-=- Low Level Design modified. --- /tmp/wklog.24.old.21663 2009-06-20 09:34:48.000000000 +0300 +++ /tmp/wklog.24.new.21663 2009-06-20 09:34:48.000000000 +0300 @@ -4,6 +4,7 @@ 2. New implementation 2.1 New tree_and() 2.2 New tree_or() +3. Testing and required coverage </contents> 1. Current implementation overview @@ -240,3 +241,14 @@ In order to limit the impact of this combinatorial explosion, we will introduce a rule that we won't generate more than #defined MAX_IMERGE_OPTS options. + +3. Testing and required coverage +================================ +So far could find the following user cases: + +* BUG#17259: Query optimizer chooses wrong index +* BUG#17673: Optimizer does not use Index Merge optimization in some cases +* BUG#23322: Optimizer sometimes erroniously prefers other index over index merge +* BUG#30151: optimizer is very reluctant to chose index_merge algorithm + + -=-=(Guest - Thu, 18 Jun 2009, 16:55)=-=- Low Level Design modified. --- /tmp/wklog.24.old.19152 2009-06-18 16:55:00.000000000 +0300 +++ /tmp/wklog.24.new.19152 2009-06-18 16:55:00.000000000 +0300 @@ -141,13 +141,15 @@ Operations on SEL_ARG trees will be modified to produce/process the trees of this kind: + 2.1 New tree_and() ------------------ In order not to lose plans, we'll make these changes: -1. Don't remove index_merge part of the tree. +A1. Don't remove index_merge part of the tree (this will take care of + DISCARD-IMERGE-1 problem) -2. Push range conditions down into index_merge trees that may support them. +A2. Push range conditions down into index_merge trees that may support them. if one tree has range(key1) and the other tree has imerge(key1 OR key2) then perform an equvalent of this operation: @@ -155,8 +157,86 @@ (rangeA(key1) AND rangeB(key1)) OR (rangeA(key1) AND rangeB(key2)) -3. Just as before: if both sel_tree A and sel_tree B have index_merge options, +A3. Just as before: if both sel_tree A and sel_tree B have index_merge options, concatenate them together. -2.2 New tree_or() +2.2 New tree_or() +----------------- +O1. Dont remove non-range plans: + Current tree_or() code will refuse to produce index_merge plans for + conditions like + + "t.key1part2=const OR t.key2part1=const" + + (this is marked as DISCARD-IMERGE-3). This was justifed as the left part of + the AND condition is not usable for range access, and the operation of + tree_and() guaranteed that there was no way it could changed to make a + usable range plan. With new tree_and() and rule A2, this is no longer the + case. For example for this query: + + (t.key1part2=const OR t.key2part1=const) AND t.key1part1=const + + it will construct a + + imerge(t.key1part2=const OR t.key2part1=const), range(t.key1part1=const) + + then tree_and() will apply rule A2 to push the range down into index merge + and after that we'll have: + + range(t.key1part1=const) + imerge( + t.key1part2=const AND t.key1part1=const, + t.key2part1=const + ) + note that imerge(...) describes a usable index_merge plan and it's possible + that it will be the best access path. + +O2. "Create index_merge accesses when possible" + Current tree_or() will not create index_merge access when it could create + non-index merge access (see DISCARD-IMERGE-3 and its example in the "Problems + in the current implementation" section). This will be changed to work as + follows: we will create index_merge made for index scans that didn't have + their match in the other sel_tree. + Ilustrating it with an example: + + | sel_tree_A | sel_tree_B | A or B | include in index_merge? + ------+------------+------------+--------+------------------------ + key1 | cond1 | cond2 | condM | no + key2 | cond3 | cond4 | NULL | no + key3 | cond5 | | | yes, A-side + key4 | cond6 | | | yes, A-side + key5 | | cond7 | | yes, B-side + key6 | | cond8 | | yes, B-side + + here we assume that + - (cond1 OR cond2) did produce a combined range. Not including them in + index_merge. + - (cond3 OR cond4) didn't produce a usable range (e.g. they were + t.key1part1=c1 AND t.key1part2=c1, respectively, and combining them + didn't yield any range list) + - All other scand didn't have their counterparts, so we'll end up with a + SEL_TREE of: + + range(condM) AND index_merge((cond5 AND cond6),(cond7 AND cond8)) + . + +O4. There is no O4. DISCARD-INDEX-MERGE-4 will remain there. The idea is +that although DISCARD-INDEX-MERGE-4 does discard plans, so far we haven +seen any complaints that could be attributed to it. +If we face the need to lift DISCARD-INDEX-MERGE-4, our answer will be to +lift it ,and produce a cross-product: + + ((key1p OR key2p) AND (key3p OR key4p)) + OR + ((key5p OR key6p) AND (key7p OR key8p)) + + = (key1p OR key2p OR key5p OR key6p) AND // this part is currently + (key3p OR key4p OR key5p OR key6p) AND // produced + + (key1p OR key2p OR key5p OR key6p) AND // this part will be added + (key3p OR key4p OR key5p OR key6p) //. + +In order to limit the impact of this combinatorial explosion, we will +introduce a rule that we won't generate more than #defined +MAX_IMERGE_OPTS options. -=-=(Guest - Thu, 18 Jun 2009, 14:56)=-=- Low Level Design modified. --- /tmp/wklog.24.old.15612 2009-06-18 14:56:09.000000000 +0300 +++ /tmp/wklog.24.new.15612 2009-06-18 14:56:09.000000000 +0300 @@ -1 +1,162 @@ +<contents> +1. Current implementation overview +1.1. Problems in the current implementation +2. New implementation +2.1 New tree_and() +2.2 New tree_or() +</contents> + +1. Current implementation overview +================================== +At the moment, range analyzer works as follows: + +SEL_TREE structure represents + + # There are sel_trees, a sel_tree is either range or merge tree + sel_tree = range_tree | imerge_tree + + # a range tree has range access options, possibly for several keys + range_tree = range(key1) AND range(key2) AND ... AND range(keyN); + + # merge tree represents several way to index_merge + imerge_tree = imerge1 AND imerge2 AND ... + + # a way to do index merge == a set to use of different indexes. + imergeX = range_tree1 OR range_tree2 OR .. + where no pair of range_treeX have ranges over the same index. + + + tree_and(A, B) + { + if (both A and B are range trees) + return a range_tree with computed intersection for each range; + if (only one of A and B is a range tree) + return that tree; // DISCARD-IMERGE-1 + // at this point both trees are index_merge trees + return concat_lists( A.imerge1 ... A.imergeN, B.imerge1 ... B.imergeN); + } + + + tree_or(A, B) + { + if (A and B are range trees) + { + R = new range_tree; + for each index i + R.add(range_union(A.range(i), B.range(i))); + + if (R has at least one range access) + return R; + else + { + /* could not build any range accesses. construct index_merge */ + remove non-ranges from A; // DISCARD-IMERGE-2 + remove non-ranges from B; + return new index_merge(A, B); + } + } + else if (A is range tree and B is index_merge tree (or vice versa)) + { + Perform this transformation: + + range_treeA // this is A + OR + (range_treeB_11 OR range_treeB_12 OR ... OR range_treeB_1N) AND + (range_treeB_21 OR range_treeB_22 OR ... OR range_treeB_2N) AND + ... + (range_treeB_K1 OR range_treeB_K2 OR ... OR range_treeB_kN) AND + = + (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND + (range_treeA OR range_treeB_21 OR ... OR range_treeB_2N) AND + ... + (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND + + Now each line represents an index_merge.. + } + else if (both A and B are index_merge trees) + { + Perform this transformation: + + imergeA1 AND imergeA2 AND ... AND imergeAN + OR + imergeB1 AND imergeB2 AND ... AND imergeBN + + -> (discard all imergeA{i=2,3,...} -> // DISCARD-IMERGE-3 + + imergeA1 + OR + imergeB1 AND imergeB2 AND ... AND imergeBN = + + = (combine imergeA1 with each of the imergeB{i} ) = + + combine(imergeA1 OR imergeB1) AND + combine(imergeA1 OR imergeB2) AND + ... AND + combine(imergeA1 OR imergeBN) + } + } + +1.1. Problems in the current implementation +------------------------------------------- +As marked in the code above: + +DISCARD-IMERGE-1 step will cause index_merge option to be discarded when +the WHERE clause has this form: + + (t.key1=c1 OR t.key2=c2) AND t.badkey < c3 + +DISCARD-IMERGE-2 step will cause index_merge option to be discarded when +the WHERE clause has this form (conditions t.badkey may have abritrary form): + + (t.badkey<c1 AND t.key1=c1) OR (t.key1=c2 AND t.badkey < c2) + +DISCARD-IMERGE-3 manifests itself as the following effect: suppose there are +two indexes: + + INDEX i1(col1, col2), + INDEX i2(col1, col3) + +and this WHERE clause: + + col1=c1 AND (col2=c2 OR col3=c3) + +The optimizer will generate the plans that only use the "col1=c1" part. The +right side of the AND will be ignored even if it has good selectivity. + + +2. New implementation +===================== + +<general idea> +* Don't start fighting combinatorial explosion until we've actually got one. +</> + +SEL_TREE structure will be now able to hold both index_merge and range scan +candidates at the same time. That is, + + sel_tree2 = range_tree AND imerge_tree + +where both parts are optional (i.e. can be empty) + +Operations on SEL_ARG trees will be modified to produce/process the trees of +this kind: + +2.1 New tree_and() +------------------ +In order not to lose plans, we'll make these changes: + +1. Don't remove index_merge part of the tree. + +2. Push range conditions down into index_merge trees that may support them. + if one tree has range(key1) and the other tree has imerge(key1 OR key2) + then perform an equvalent of this operation: + + rangeA(key1) AND ( rangeB(key1) OR rangeB(key2)) = + + (rangeA(key1) AND rangeB(key1)) OR (rangeA(key1) AND rangeB(key2)) + +3. Just as before: if both sel_tree A and sel_tree B have index_merge options, + concatenate them together. + +2.2 New tree_or() -=-=(Psergey - Wed, 03 Jun 2009, 12:09)=-=- Dependency created: 30 now depends on 24 -=-=(Guest - Mon, 01 Jun 2009, 23:30)=-=- High-Level Specification modified. --- /tmp/wklog.24.old.21580 2009-06-01 23:30:06.000000000 +0300 +++ /tmp/wklog.24.new.21580 2009-06-01 23:30:06.000000000 +0300 @@ -64,6 +64,9 @@ * How strict is the limitation on the form of the WHERE? +* Which version should this be based on? 5.1? Which patches are should be in + (google's/percona's/maria/etc?) + * TODO: The optimizer didn't compare costs of index_merge and range before (ok it did but that was done for accesses to different tables). Will there be any possible gotchas here? -=-=(Guest - Wed, 27 May 2009, 13:59)=-=- Title modified. --- /tmp/wklog.24.old.9498 2009-05-27 13:59:23.000000000 +0300 +++ /tmp/wklog.24.new.9498 2009-05-27 13:59:23.000000000 +0300 @@ -1 +1 @@ -index_merge optimizer: dont discard index_merge union strategies when range is available +index_merge: fair choice between index_merge union and range access -=-=(Guest - Tue, 26 May 2009, 13:27)=-=- High-Level Specification modified. --- /tmp/wklog.24.old.305 2009-05-26 13:27:32.000000000 +0300 +++ /tmp/wklog.24.new.305 2009-05-26 13:27:32.000000000 +0300 @@ -1 +1,70 @@ +(Not a ready HLS but draft) +<contents> +Solution overview +Limitations +TODO + +</contents> + +Solution overview +================= +The idea is to delay discarding potential index_merge plans until the point +where it is really necessary. + +This way, we won't have to do much changes in the range analyzer, but will be +able to keep potential index_merge plan just enough so that it's possible to +take it into consideration together with range access plans. + +Since there are no changes in the optimizer, the ability to consider both +range and index_merge options will be limited to WHERE clauses of this form: + + WHERE := range_cond(key1_1) AND + range_cond(key2_1) AND + other_cond AND + index_merge_OR_cond1(key3_1, key3_2, ...) + index_merge_OR_cond2(key4_1, key4_2, ...) + +where + + index_merge_OR_cond{N} := (range_cond(keyN_1) OR + range_cond(keyN_2) OR ...) + + + range_cond(keyX) := condition that allows to construct range access of keyX + and doesn't allow to construct range/index_merge accesses + for any keys of the table in question. + + +For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: + + SEL_TREE( + range(key1_1), + ... + range(key2_1), + SEL_IMERGE( (1) + SEL_TREE(key3_1}) + SEL_TREE(key3_2}) + ... + ) + ... + ) + +which can be used to make a cost-based choice between range and index_merge. + +Limitations +----------- +This will not be a full solution in a sense that the range analyzer will not +be able to produce sel_tree (1) if the WHERE clause is specified in other form +(e.g. brackets were opened). + +TODO +---- +* is it a problem if there are keys that are referred to both from + index_merge and from range access? + +* How strict is the limitation on the form of the WHERE? + +* TODO: The optimizer didn't compare costs of index_merge and range before (ok + it did but that was done for accesses to different tables). Will there be any + possible gotchas here? DESCRIPTION: Current range optimizer will discard possible index_merge/[sort]union strategies when there is a possible range plan. This action is a part of measures we take to avoid combinatorial explosion of possible range/ index_merge strategies. A bad side effect of this is that for WHERE clauses in form t.key1= 'very-frequent-value' AND (t.key2='rare-value1' OR t.key3='rare-value2') the optimizer will - discard union(key2,key3) in favor of range(key1) - consider costs of using range(key1) and discard that plan also and the overall effect is that possible poor range access will cause possible good index_merge access not to be considered. This WL is to about lifting this limitation at least for some subset of WHERE clauses. HIGH-LEVEL SPECIFICATION: (Not a ready HLS but draft) <contents> Solution overview Limitations TODO </contents> Solution overview ================= The idea is to delay discarding potential index_merge plans until the point where it is really necessary. This way, we won't have to do much changes in the range analyzer, but will be able to keep potential index_merge plan just enough so that it's possible to take it into consideration together with range access plans. Since there are no changes in the optimizer, the ability to consider both range and index_merge options will be limited to WHERE clauses of this form: WHERE := range_cond(key1_1) AND range_cond(key2_1) AND other_cond AND index_merge_OR_cond1(key3_1, key3_2, ...) index_merge_OR_cond2(key4_1, key4_2, ...) where index_merge_OR_cond{N} := (range_cond(keyN_1) OR range_cond(keyN_2) OR ...) range_cond(keyX) := condition that allows to construct range access of keyX and doesn't allow to construct range/index_merge accesses for any keys of the table in question. For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: SEL_TREE( range(key1_1), ... range(key2_1), SEL_IMERGE( (1) SEL_TREE(key3_1}) SEL_TREE(key3_2}) ... ) ... ) which can be used to make a cost-based choice between range and index_merge. Limitations ----------- This will not be a full solution in a sense that the range analyzer will not be able to produce sel_tree (1) if the WHERE clause is specified in other form (e.g. brackets were opened). TODO ---- * is it a problem if there are keys that are referred to both from index_merge and from range access? * How strict is the limitation on the form of the WHERE? * Which version should this be based on? 5.1? Which patches are should be in (google's/percona's/maria/etc?) * TODO: The optimizer didn't compare costs of index_merge and range before (ok it did but that was done for accesses to different tables). Will there be any possible gotchas here? LOW-LEVEL DESIGN: <contents> 1. Current implementation overview 1.1. Problems in the current implementation 2. New implementation 2.1 New tree_and() 2.2 New tree_or() 3. Testing and required coverage </contents> 1. Current implementation overview ================================== At the moment, range analyzer works as follows: SEL_TREE structure represents # There are sel_trees, a sel_tree is either range or merge tree sel_tree = range_tree | imerge_tree # a range tree has range access options, possibly for several keys range_tree = range(key1) AND range(key2) AND ... AND range(keyN); (here range(keyi) may represent ranges not for initial keyi prefixes, but ranges for any infixes for keyi) # merge tree represents several way to index_merge imerge_tree = imerge1 AND imerge2 AND ... # a way to do index merge == a set to use of different indexes. imergeX = range_tree1 OR range_tree2 OR .. where no pair of range_treeX have ranges over the same index. tree_and(A, B) { if (both A and B are range trees) return a range_tree with computed intersection for each range; if (only one of A and B is a range tree) return that tree; // DISCARD-IMERGE-1 // at this point both trees are index_merge trees return concat_lists( A.imerge1 ... A.imergeN, B.imerge1 ... B.imergeN); } tree_or(A, B) { if (A and B are range trees) { R = new range_tree; for each index i R.add(range_union(A.range(i), B.range(i))); if (R has at least one range access) return R; // DISCARD-IMERGE-2 else { /* could not build any range accesses. construct index_merge */ remove non-ranges from A; remove non-ranges from B; return new index_merge(A, B); // DISCARD-IMERGE-3 } } else if (A is range tree and B is index_merge tree (or vice versa)) { Perform this transformation: range_treeA // this is A OR (range_treeB_11 OR range_treeB_12 OR ... OR range_treeB_1N) AND (range_treeB_21 OR range_treeB_22 OR ... OR range_treeB_2N) AND ... (range_treeB_K1 OR range_treeB_K2 OR ... OR range_treeB_kN) = (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND (range_treeA OR range_treeB_21 OR ... OR range_treeB_2N) AND ... (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) Now each line represents an index_merge.. } else if (both A and B are index_merge trees) { Perform this transformation: imergeA1 AND imergeA2 AND ... AND imergeAN OR imergeB1 AND imergeB2 AND ... AND imergeBN -> (discard all imergeA{i=2,3,...} -> // DISCARD-IMERGE-4 imergeA1 OR imergeB1 = = (combine imergeA1 with each of the range_treeB_1{i} ) = combine(imergeA1 OR range_treeB_11) AND combine(imergeA1 OR range_treeB_12) AND ... AND combine(imergeA1 OR range_treeB_1N) } } 1.1. Problems in the current implementation ------------------------------------------- As marked in the code above: DISCARD-IMERGE-1 step will cause index_merge option to be discarded when the WHERE clause has this form: (t.key1=c1 OR t.key2=c2) AND t.badkey < c3 DISCARD-IMERGE-2 step will cause index_merge option to be discarded when the WHERE clause has this form (conditions t.badkey may have abritrary form): (t.badkey<c1 AND t.key1=c1) OR (t.key2=c2 AND t.badkey < c2) DISCARD-IMERGE-3 manifests itself as the following effect: suppose there are two indexes: INDEX i1(col1, col2), INDEX i2(col1, col3) and this WHERE clause: col1=c1 AND (col2=c2 OR col3=c3) The optimizer will generate the plans that only use the "col1=c1" part. The right side of the AND will be ignored even if it has good selectivity. (Here an imerge for col2=c2 OR col3=c3 won't be built since neither col2=c2 nor col3=c3 represent index ranges.) 2. New implementation ===================== <general idea> * Don't start fighting combinatorial explosion until we've actually got one. </> SEL_TREE structure will be now able to hold both index_merge and range scan candidates at the same time. That is, sel_tree2 = range_tree AND imerge_tree where both parts are optional (i.e. can be empty) Operations on SEL_ARG trees will be modified to produce/process the trees of this kind: 2.1 New tree_and() ------------------ In order not to lose plans, we'll make these changes: A1. Don't remove index_merge part of the tree (this will take care of DISCARD-IMERGE-1 problem) A2. Push range conditions down into index_merge trees that may support them. if one tree has range(key1) and the other tree has imerge(key1 OR key2) then perform an equvalent of this operation: rangeA(key1) AND ( rangeB(key1) OR rangeB(key2)) = (rangeA(key1) AND rangeB(key1)) OR (rangeA(key1) AND rangeB(key2)) A3. Just as before: if both sel_tree A and sel_tree B have index_merge options, concatenate them together. 2.2 New tree_or() ----------------- O1. Dont remove non-range plans: Current tree_or() code will refuse to produce index_merge plans for conditions like "t.key1part2=const OR t.key2part1=const" (this is marked as DISCARD-IMERGE-3). This was justifed as the left part of the AND condition is not usable for range access, and the operation of tree_and() guaranteed that there was no way it could changed to make a usable range plan. With new tree_and() and rule A2, this is no longer the case. For example for this query: (t.key1part2=const OR t.key2part1=const) AND t.key1part1=const it will construct a imerge(t.key1part2=const OR t.key2part1=const), range(t.key1part1=const) then tree_and() will apply rule A2 to push the range down into index merge and after that we'll have: range(t.key1part1=const) imerge( t.key1part2=const AND t.key1part1=const, t.key2part1=const ) note that imerge(...) describes a usable index_merge plan and it's possible that it will be the best access path. O2. "Create index_merge accesses when possible" Current tree_or() will not create index_merge access when it could create non-index merge access (see DISCARD-IMERGE-3 and its example in the "Problems in the current implementation" section). This will be changed to work as follows: we will create index_merge made for index scans that didn't have their match in the other sel_tree. Ilustrating it with an example: | sel_tree_A | sel_tree_B | A or B | include in index_merge? ------+------------+------------+--------+------------------------ key1 | cond1 | cond2 | condM | no key2 | cond3 | cond4 | NULL | no key3 | cond5 | | | yes, A-side key4 | cond6 | | | yes, A-side key5 | | cond7 | | yes, B-side key6 | | cond8 | | yes, B-side here we assume that - (cond1 OR cond2) did produce a combined range. Not including them in index_merge. - (cond3 OR cond4) didn't produce a usable range (e.g. they were t.key1part1=c1 AND t.key1part2=c1, respectively, and combining them didn't yield any range list) - All other scand didn't have their counterparts, so we'll end up with a SEL_TREE of: range(condM) AND index_merge((cond5 AND cond6),(cond7 AND cond8)) . O4. There is no O4. DISCARD-INDEX-MERGE-4 will remain there. The idea is that although DISCARD-INDEX-MERGE-4 does discard plans, so far we haven seen any complaints that could be attributed to it. If we face the need to lift DISCARD-INDEX-MERGE-4, our answer will be to lift it ,and produce a cross-product: ((key1p OR key2p) AND (key3p OR key4p)) OR ((key5p OR key6p) AND (key7p OR key8p)) = (key1p OR key2p OR key5p OR key6p) AND // this part is currently (key3p OR key4p OR key5p OR key6p) AND // produced (key1p OR key2p OR key5p OR key6p) AND // this part will be added (key3p OR key4p OR key5p OR key6p) //. In order to limit the impact of this combinatorial explosion, we will introduce a rule that we won't generate more than #defined MAX_IMERGE_OPTS options. 3. Testing and required coverage ================================ So far could find the following user cases: * BUG#17259: Query optimizer chooses wrong index * BUG#17673: Optimizer does not use Index Merge optimization in some cases * BUG#23322: Optimizer sometimes erroniously prefers other index over index merge * BUG#30151: optimizer is very reluctant to chose index_merge algorithm ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): index_merge: fair choice between index_merge union and range access (24)
by worklog-noreply＠askmonty.org 16 Aug '09

16 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: index_merge: fair choice between index_merge union and range access CREATION DATE..: Tue, 26 May 2009, 12:10 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: Psergey CATEGORY.......: Server-Sprint TASK ID........: 24 (http://askmonty.org/worklog/?tid=24) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Sun, 16 Aug 2009, 01:03)=-=- Low Level Design modified. --- /tmp/wklog.24.old.20767 2009-08-16 01:03:11.000000000 +0300 +++ /tmp/wklog.24.new.20767 2009-08-16 01:03:11.000000000 +0300 @@ -18,6 +18,8 @@ # a range tree has range access options, possibly for several keys range_tree = range(key1) AND range(key2) AND ... AND range(keyN); + (here range(keyi) may represent ranges not for initial keyi prefixes, + but ranges for any infixes for keyi) # merge tree represents several way to index_merge imerge_tree = imerge1 AND imerge2 AND ... @@ -47,13 +49,13 @@ R.add(range_union(A.range(i), B.range(i))); if (R has at least one range access) - return R; + return R; // DISCARD-IMERGE-2 else { /* could not build any range accesses. construct index_merge */ - remove non-ranges from A; // DISCARD-IMERGE-2 + remove non-ranges from A; remove non-ranges from B; - return new index_merge(A, B); + return new index_merge(A, B); // DISCARD-IMERGE-3 } } else if (A is range tree and B is index_merge tree (or vice versa)) @@ -65,12 +67,12 @@ (range_treeB_11 OR range_treeB_12 OR ... OR range_treeB_1N) AND (range_treeB_21 OR range_treeB_22 OR ... OR range_treeB_2N) AND ... - (range_treeB_K1 OR range_treeB_K2 OR ... OR range_treeB_kN) AND + (range_treeB_K1 OR range_treeB_K2 OR ... OR range_treeB_kN) = (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND (range_treeA OR range_treeB_21 OR ... OR range_treeB_2N) AND ... - (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND + (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) Now each line represents an index_merge.. } @@ -82,18 +84,18 @@ OR imergeB1 AND imergeB2 AND ... AND imergeBN - -> (discard all imergeA{i=2,3,...} -> // DISCARD-IMERGE-3 + -> (discard all imergeA{i=2,3,...} -> // DISCARD-IMERGE-4 imergeA1 OR - imergeB1 AND imergeB2 AND ... AND imergeBN = + imergeB1 = - = (combine imergeA1 with each of the imergeB{i} ) = + = (combine imergeA1 with each of the range_treeB_1{i} ) = - combine(imergeA1 OR imergeB1) AND - combine(imergeA1 OR imergeB2) AND + combine(imergeA1 OR range_treeB_11) AND + combine(imergeA1 OR range_treeB_12) AND ... AND - combine(imergeA1 OR imergeBN) + combine(imergeA1 OR range_treeB_1N) } } @@ -109,7 +111,7 @@ DISCARD-IMERGE-2 step will cause index_merge option to be discarded when the WHERE clause has this form (conditions t.badkey may have abritrary form): - (t.badkey<c1 AND t.key1=c1) OR (t.key1=c2 AND t.badkey < c2) + (t.badkey<c1 AND t.key1=c1) OR (t.key2=c2 AND t.badkey < c2) DISCARD-IMERGE-3 manifests itself as the following effect: suppose there are two indexes: @@ -123,6 +125,8 @@ The optimizer will generate the plans that only use the "col1=c1" part. The right side of the AND will be ignored even if it has good selectivity. +(Here an imerge for col2=c2 OR col3=c3 won't be built since neither col2=c2 nor +col3=c3 represent index ranges.) 2. New implementation -=-=(Guest - Mon, 20 Jul 2009, 17:13)=-=- Dependency deleted: 30 no longer depends on 24 -=-=(Guest - Sat, 20 Jun 2009, 09:34)=-=- Low Level Design modified. --- /tmp/wklog.24.old.21663 2009-06-20 09:34:48.000000000 +0300 +++ /tmp/wklog.24.new.21663 2009-06-20 09:34:48.000000000 +0300 @@ -4,6 +4,7 @@ 2. New implementation 2.1 New tree_and() 2.2 New tree_or() +3. Testing and required coverage </contents> 1. Current implementation overview @@ -240,3 +241,14 @@ In order to limit the impact of this combinatorial explosion, we will introduce a rule that we won't generate more than #defined MAX_IMERGE_OPTS options. + +3. Testing and required coverage +================================ +So far could find the following user cases: + +* BUG#17259: Query optimizer chooses wrong index +* BUG#17673: Optimizer does not use Index Merge optimization in some cases +* BUG#23322: Optimizer sometimes erroniously prefers other index over index merge +* BUG#30151: optimizer is very reluctant to chose index_merge algorithm + + -=-=(Guest - Thu, 18 Jun 2009, 16:55)=-=- Low Level Design modified. --- /tmp/wklog.24.old.19152 2009-06-18 16:55:00.000000000 +0300 +++ /tmp/wklog.24.new.19152 2009-06-18 16:55:00.000000000 +0300 @@ -141,13 +141,15 @@ Operations on SEL_ARG trees will be modified to produce/process the trees of this kind: + 2.1 New tree_and() ------------------ In order not to lose plans, we'll make these changes: -1. Don't remove index_merge part of the tree. +A1. Don't remove index_merge part of the tree (this will take care of + DISCARD-IMERGE-1 problem) -2. Push range conditions down into index_merge trees that may support them. +A2. Push range conditions down into index_merge trees that may support them. if one tree has range(key1) and the other tree has imerge(key1 OR key2) then perform an equvalent of this operation: @@ -155,8 +157,86 @@ (rangeA(key1) AND rangeB(key1)) OR (rangeA(key1) AND rangeB(key2)) -3. Just as before: if both sel_tree A and sel_tree B have index_merge options, +A3. Just as before: if both sel_tree A and sel_tree B have index_merge options, concatenate them together. -2.2 New tree_or() +2.2 New tree_or() +----------------- +O1. Dont remove non-range plans: + Current tree_or() code will refuse to produce index_merge plans for + conditions like + + "t.key1part2=const OR t.key2part1=const" + + (this is marked as DISCARD-IMERGE-3). This was justifed as the left part of + the AND condition is not usable for range access, and the operation of + tree_and() guaranteed that there was no way it could changed to make a + usable range plan. With new tree_and() and rule A2, this is no longer the + case. For example for this query: + + (t.key1part2=const OR t.key2part1=const) AND t.key1part1=const + + it will construct a + + imerge(t.key1part2=const OR t.key2part1=const), range(t.key1part1=const) + + then tree_and() will apply rule A2 to push the range down into index merge + and after that we'll have: + + range(t.key1part1=const) + imerge( + t.key1part2=const AND t.key1part1=const, + t.key2part1=const + ) + note that imerge(...) describes a usable index_merge plan and it's possible + that it will be the best access path. + +O2. "Create index_merge accesses when possible" + Current tree_or() will not create index_merge access when it could create + non-index merge access (see DISCARD-IMERGE-3 and its example in the "Problems + in the current implementation" section). This will be changed to work as + follows: we will create index_merge made for index scans that didn't have + their match in the other sel_tree. + Ilustrating it with an example: + + | sel_tree_A | sel_tree_B | A or B | include in index_merge? + ------+------------+------------+--------+------------------------ + key1 | cond1 | cond2 | condM | no + key2 | cond3 | cond4 | NULL | no + key3 | cond5 | | | yes, A-side + key4 | cond6 | | | yes, A-side + key5 | | cond7 | | yes, B-side + key6 | | cond8 | | yes, B-side + + here we assume that + - (cond1 OR cond2) did produce a combined range. Not including them in + index_merge. + - (cond3 OR cond4) didn't produce a usable range (e.g. they were + t.key1part1=c1 AND t.key1part2=c1, respectively, and combining them + didn't yield any range list) + - All other scand didn't have their counterparts, so we'll end up with a + SEL_TREE of: + + range(condM) AND index_merge((cond5 AND cond6),(cond7 AND cond8)) + . + +O4. There is no O4. DISCARD-INDEX-MERGE-4 will remain there. The idea is +that although DISCARD-INDEX-MERGE-4 does discard plans, so far we haven +seen any complaints that could be attributed to it. +If we face the need to lift DISCARD-INDEX-MERGE-4, our answer will be to +lift it ,and produce a cross-product: + + ((key1p OR key2p) AND (key3p OR key4p)) + OR + ((key5p OR key6p) AND (key7p OR key8p)) + + = (key1p OR key2p OR key5p OR key6p) AND // this part is currently + (key3p OR key4p OR key5p OR key6p) AND // produced + + (key1p OR key2p OR key5p OR key6p) AND // this part will be added + (key3p OR key4p OR key5p OR key6p) //. + +In order to limit the impact of this combinatorial explosion, we will +introduce a rule that we won't generate more than #defined +MAX_IMERGE_OPTS options. -=-=(Guest - Thu, 18 Jun 2009, 14:56)=-=- Low Level Design modified. --- /tmp/wklog.24.old.15612 2009-06-18 14:56:09.000000000 +0300 +++ /tmp/wklog.24.new.15612 2009-06-18 14:56:09.000000000 +0300 @@ -1 +1,162 @@ +<contents> +1. Current implementation overview +1.1. Problems in the current implementation +2. New implementation +2.1 New tree_and() +2.2 New tree_or() +</contents> + +1. Current implementation overview +================================== +At the moment, range analyzer works as follows: + +SEL_TREE structure represents + + # There are sel_trees, a sel_tree is either range or merge tree + sel_tree = range_tree | imerge_tree + + # a range tree has range access options, possibly for several keys + range_tree = range(key1) AND range(key2) AND ... AND range(keyN); + + # merge tree represents several way to index_merge + imerge_tree = imerge1 AND imerge2 AND ... + + # a way to do index merge == a set to use of different indexes. + imergeX = range_tree1 OR range_tree2 OR .. + where no pair of range_treeX have ranges over the same index. + + + tree_and(A, B) + { + if (both A and B are range trees) + return a range_tree with computed intersection for each range; + if (only one of A and B is a range tree) + return that tree; // DISCARD-IMERGE-1 + // at this point both trees are index_merge trees + return concat_lists( A.imerge1 ... A.imergeN, B.imerge1 ... B.imergeN); + } + + + tree_or(A, B) + { + if (A and B are range trees) + { + R = new range_tree; + for each index i + R.add(range_union(A.range(i), B.range(i))); + + if (R has at least one range access) + return R; + else + { + /* could not build any range accesses. construct index_merge */ + remove non-ranges from A; // DISCARD-IMERGE-2 + remove non-ranges from B; + return new index_merge(A, B); + } + } + else if (A is range tree and B is index_merge tree (or vice versa)) + { + Perform this transformation: + + range_treeA // this is A + OR + (range_treeB_11 OR range_treeB_12 OR ... OR range_treeB_1N) AND + (range_treeB_21 OR range_treeB_22 OR ... OR range_treeB_2N) AND + ... + (range_treeB_K1 OR range_treeB_K2 OR ... OR range_treeB_kN) AND + = + (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND + (range_treeA OR range_treeB_21 OR ... OR range_treeB_2N) AND + ... + (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND + + Now each line represents an index_merge.. + } + else if (both A and B are index_merge trees) + { + Perform this transformation: + + imergeA1 AND imergeA2 AND ... AND imergeAN + OR + imergeB1 AND imergeB2 AND ... AND imergeBN + + -> (discard all imergeA{i=2,3,...} -> // DISCARD-IMERGE-3 + + imergeA1 + OR + imergeB1 AND imergeB2 AND ... AND imergeBN = + + = (combine imergeA1 with each of the imergeB{i} ) = + + combine(imergeA1 OR imergeB1) AND + combine(imergeA1 OR imergeB2) AND + ... AND + combine(imergeA1 OR imergeBN) + } + } + +1.1. Problems in the current implementation +------------------------------------------- +As marked in the code above: + +DISCARD-IMERGE-1 step will cause index_merge option to be discarded when +the WHERE clause has this form: + + (t.key1=c1 OR t.key2=c2) AND t.badkey < c3 + +DISCARD-IMERGE-2 step will cause index_merge option to be discarded when +the WHERE clause has this form (conditions t.badkey may have abritrary form): + + (t.badkey<c1 AND t.key1=c1) OR (t.key1=c2 AND t.badkey < c2) + +DISCARD-IMERGE-3 manifests itself as the following effect: suppose there are +two indexes: + + INDEX i1(col1, col2), + INDEX i2(col1, col3) + +and this WHERE clause: + + col1=c1 AND (col2=c2 OR col3=c3) + +The optimizer will generate the plans that only use the "col1=c1" part. The +right side of the AND will be ignored even if it has good selectivity. + + +2. New implementation +===================== + +<general idea> +* Don't start fighting combinatorial explosion until we've actually got one. +</> + +SEL_TREE structure will be now able to hold both index_merge and range scan +candidates at the same time. That is, + + sel_tree2 = range_tree AND imerge_tree + +where both parts are optional (i.e. can be empty) + +Operations on SEL_ARG trees will be modified to produce/process the trees of +this kind: + +2.1 New tree_and() +------------------ +In order not to lose plans, we'll make these changes: + +1. Don't remove index_merge part of the tree. + +2. Push range conditions down into index_merge trees that may support them. + if one tree has range(key1) and the other tree has imerge(key1 OR key2) + then perform an equvalent of this operation: + + rangeA(key1) AND ( rangeB(key1) OR rangeB(key2)) = + + (rangeA(key1) AND rangeB(key1)) OR (rangeA(key1) AND rangeB(key2)) + +3. Just as before: if both sel_tree A and sel_tree B have index_merge options, + concatenate them together. + +2.2 New tree_or() -=-=(Psergey - Wed, 03 Jun 2009, 12:09)=-=- Dependency created: 30 now depends on 24 -=-=(Guest - Mon, 01 Jun 2009, 23:30)=-=- High-Level Specification modified. --- /tmp/wklog.24.old.21580 2009-06-01 23:30:06.000000000 +0300 +++ /tmp/wklog.24.new.21580 2009-06-01 23:30:06.000000000 +0300 @@ -64,6 +64,9 @@ * How strict is the limitation on the form of the WHERE? +* Which version should this be based on? 5.1? Which patches are should be in + (google's/percona's/maria/etc?) + * TODO: The optimizer didn't compare costs of index_merge and range before (ok it did but that was done for accesses to different tables). Will there be any possible gotchas here? -=-=(Guest - Wed, 27 May 2009, 13:59)=-=- Title modified. --- /tmp/wklog.24.old.9498 2009-05-27 13:59:23.000000000 +0300 +++ /tmp/wklog.24.new.9498 2009-05-27 13:59:23.000000000 +0300 @@ -1 +1 @@ -index_merge optimizer: dont discard index_merge union strategies when range is available +index_merge: fair choice between index_merge union and range access -=-=(Guest - Tue, 26 May 2009, 13:27)=-=- High-Level Specification modified. --- /tmp/wklog.24.old.305 2009-05-26 13:27:32.000000000 +0300 +++ /tmp/wklog.24.new.305 2009-05-26 13:27:32.000000000 +0300 @@ -1 +1,70 @@ +(Not a ready HLS but draft) +<contents> +Solution overview +Limitations +TODO + +</contents> + +Solution overview +================= +The idea is to delay discarding potential index_merge plans until the point +where it is really necessary. + +This way, we won't have to do much changes in the range analyzer, but will be +able to keep potential index_merge plan just enough so that it's possible to +take it into consideration together with range access plans. + +Since there are no changes in the optimizer, the ability to consider both +range and index_merge options will be limited to WHERE clauses of this form: + + WHERE := range_cond(key1_1) AND + range_cond(key2_1) AND + other_cond AND + index_merge_OR_cond1(key3_1, key3_2, ...) + index_merge_OR_cond2(key4_1, key4_2, ...) + +where + + index_merge_OR_cond{N} := (range_cond(keyN_1) OR + range_cond(keyN_2) OR ...) + + + range_cond(keyX) := condition that allows to construct range access of keyX + and doesn't allow to construct range/index_merge accesses + for any keys of the table in question. + + +For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: + + SEL_TREE( + range(key1_1), + ... + range(key2_1), + SEL_IMERGE( (1) + SEL_TREE(key3_1}) + SEL_TREE(key3_2}) + ... + ) + ... + ) + +which can be used to make a cost-based choice between range and index_merge. + +Limitations +----------- +This will not be a full solution in a sense that the range analyzer will not +be able to produce sel_tree (1) if the WHERE clause is specified in other form +(e.g. brackets were opened). + +TODO +---- +* is it a problem if there are keys that are referred to both from + index_merge and from range access? + +* How strict is the limitation on the form of the WHERE? + +* TODO: The optimizer didn't compare costs of index_merge and range before (ok + it did but that was done for accesses to different tables). Will there be any + possible gotchas here? DESCRIPTION: Current range optimizer will discard possible index_merge/[sort]union strategies when there is a possible range plan. This action is a part of measures we take to avoid combinatorial explosion of possible range/ index_merge strategies. A bad side effect of this is that for WHERE clauses in form t.key1= 'very-frequent-value' AND (t.key2='rare-value1' OR t.key3='rare-value2') the optimizer will - discard union(key2,key3) in favor of range(key1) - consider costs of using range(key1) and discard that plan also and the overall effect is that possible poor range access will cause possible good index_merge access not to be considered. This WL is to about lifting this limitation at least for some subset of WHERE clauses. HIGH-LEVEL SPECIFICATION: (Not a ready HLS but draft) <contents> Solution overview Limitations TODO </contents> Solution overview ================= The idea is to delay discarding potential index_merge plans until the point where it is really necessary. This way, we won't have to do much changes in the range analyzer, but will be able to keep potential index_merge plan just enough so that it's possible to take it into consideration together with range access plans. Since there are no changes in the optimizer, the ability to consider both range and index_merge options will be limited to WHERE clauses of this form: WHERE := range_cond(key1_1) AND range_cond(key2_1) AND other_cond AND index_merge_OR_cond1(key3_1, key3_2, ...) index_merge_OR_cond2(key4_1, key4_2, ...) where index_merge_OR_cond{N} := (range_cond(keyN_1) OR range_cond(keyN_2) OR ...) range_cond(keyX) := condition that allows to construct range access of keyX and doesn't allow to construct range/index_merge accesses for any keys of the table in question. For such WHERE clauses, the range analyzer will produce SEL_TREE of this form: SEL_TREE( range(key1_1), ... range(key2_1), SEL_IMERGE( (1) SEL_TREE(key3_1}) SEL_TREE(key3_2}) ... ) ... ) which can be used to make a cost-based choice between range and index_merge. Limitations ----------- This will not be a full solution in a sense that the range analyzer will not be able to produce sel_tree (1) if the WHERE clause is specified in other form (e.g. brackets were opened). TODO ---- * is it a problem if there are keys that are referred to both from index_merge and from range access? * How strict is the limitation on the form of the WHERE? * Which version should this be based on? 5.1? Which patches are should be in (google's/percona's/maria/etc?) * TODO: The optimizer didn't compare costs of index_merge and range before (ok it did but that was done for accesses to different tables). Will there be any possible gotchas here? LOW-LEVEL DESIGN: <contents> 1. Current implementation overview 1.1. Problems in the current implementation 2. New implementation 2.1 New tree_and() 2.2 New tree_or() 3. Testing and required coverage </contents> 1. Current implementation overview ================================== At the moment, range analyzer works as follows: SEL_TREE structure represents # There are sel_trees, a sel_tree is either range or merge tree sel_tree = range_tree | imerge_tree # a range tree has range access options, possibly for several keys range_tree = range(key1) AND range(key2) AND ... AND range(keyN); (here range(keyi) may represent ranges not for initial keyi prefixes, but ranges for any infixes for keyi) # merge tree represents several way to index_merge imerge_tree = imerge1 AND imerge2 AND ... # a way to do index merge == a set to use of different indexes. imergeX = range_tree1 OR range_tree2 OR .. where no pair of range_treeX have ranges over the same index. tree_and(A, B) { if (both A and B are range trees) return a range_tree with computed intersection for each range; if (only one of A and B is a range tree) return that tree; // DISCARD-IMERGE-1 // at this point both trees are index_merge trees return concat_lists( A.imerge1 ... A.imergeN, B.imerge1 ... B.imergeN); } tree_or(A, B) { if (A and B are range trees) { R = new range_tree; for each index i R.add(range_union(A.range(i), B.range(i))); if (R has at least one range access) return R; // DISCARD-IMERGE-2 else { /* could not build any range accesses. construct index_merge */ remove non-ranges from A; remove non-ranges from B; return new index_merge(A, B); // DISCARD-IMERGE-3 } } else if (A is range tree and B is index_merge tree (or vice versa)) { Perform this transformation: range_treeA // this is A OR (range_treeB_11 OR range_treeB_12 OR ... OR range_treeB_1N) AND (range_treeB_21 OR range_treeB_22 OR ... OR range_treeB_2N) AND ... (range_treeB_K1 OR range_treeB_K2 OR ... OR range_treeB_kN) = (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) AND (range_treeA OR range_treeB_21 OR ... OR range_treeB_2N) AND ... (range_treeA OR range_treeB_11 OR ... OR range_treeB_1N) Now each line represents an index_merge.. } else if (both A and B are index_merge trees) { Perform this transformation: imergeA1 AND imergeA2 AND ... AND imergeAN OR imergeB1 AND imergeB2 AND ... AND imergeBN -> (discard all imergeA{i=2,3,...} -> // DISCARD-IMERGE-4 imergeA1 OR imergeB1 = = (combine imergeA1 with each of the range_treeB_1{i} ) = combine(imergeA1 OR range_treeB_11) AND combine(imergeA1 OR range_treeB_12) AND ... AND combine(imergeA1 OR range_treeB_1N) } } 1.1. Problems in the current implementation ------------------------------------------- As marked in the code above: DISCARD-IMERGE-1 step will cause index_merge option to be discarded when the WHERE clause has this form: (t.key1=c1 OR t.key2=c2) AND t.badkey < c3 DISCARD-IMERGE-2 step will cause index_merge option to be discarded when the WHERE clause has this form (conditions t.badkey may have abritrary form): (t.badkey<c1 AND t.key1=c1) OR (t.key2=c2 AND t.badkey < c2) DISCARD-IMERGE-3 manifests itself as the following effect: suppose there are two indexes: INDEX i1(col1, col2), INDEX i2(col1, col3) and this WHERE clause: col1=c1 AND (col2=c2 OR col3=c3) The optimizer will generate the plans that only use the "col1=c1" part. The right side of the AND will be ignored even if it has good selectivity. (Here an imerge for col2=c2 OR col3=c3 won't be built since neither col2=c2 nor col3=c3 represent index ranges.) 2. New implementation ===================== <general idea> * Don't start fighting combinatorial explosion until we've actually got one. </> SEL_TREE structure will be now able to hold both index_merge and range scan candidates at the same time. That is, sel_tree2 = range_tree AND imerge_tree where both parts are optional (i.e. can be empty) Operations on SEL_ARG trees will be modified to produce/process the trees of this kind: 2.1 New tree_and() ------------------ In order not to lose plans, we'll make these changes: A1. Don't remove index_merge part of the tree (this will take care of DISCARD-IMERGE-1 problem) A2. Push range conditions down into index_merge trees that may support them. if one tree has range(key1) and the other tree has imerge(key1 OR key2) then perform an equvalent of this operation: rangeA(key1) AND ( rangeB(key1) OR rangeB(key2)) = (rangeA(key1) AND rangeB(key1)) OR (rangeA(key1) AND rangeB(key2)) A3. Just as before: if both sel_tree A and sel_tree B have index_merge options, concatenate them together. 2.2 New tree_or() ----------------- O1. Dont remove non-range plans: Current tree_or() code will refuse to produce index_merge plans for conditions like "t.key1part2=const OR t.key2part1=const" (this is marked as DISCARD-IMERGE-3). This was justifed as the left part of the AND condition is not usable for range access, and the operation of tree_and() guaranteed that there was no way it could changed to make a usable range plan. With new tree_and() and rule A2, this is no longer the case. For example for this query: (t.key1part2=const OR t.key2part1=const) AND t.key1part1=const it will construct a imerge(t.key1part2=const OR t.key2part1=const), range(t.key1part1=const) then tree_and() will apply rule A2 to push the range down into index merge and after that we'll have: range(t.key1part1=const) imerge( t.key1part2=const AND t.key1part1=const, t.key2part1=const ) note that imerge(...) describes a usable index_merge plan and it's possible that it will be the best access path. O2. "Create index_merge accesses when possible" Current tree_or() will not create index_merge access when it could create non-index merge access (see DISCARD-IMERGE-3 and its example in the "Problems in the current implementation" section). This will be changed to work as follows: we will create index_merge made for index scans that didn't have their match in the other sel_tree. Ilustrating it with an example: | sel_tree_A | sel_tree_B | A or B | include in index_merge? ------+------------+------------+--------+------------------------ key1 | cond1 | cond2 | condM | no key2 | cond3 | cond4 | NULL | no key3 | cond5 | | | yes, A-side key4 | cond6 | | | yes, A-side key5 | | cond7 | | yes, B-side key6 | | cond8 | | yes, B-side here we assume that - (cond1 OR cond2) did produce a combined range. Not including them in index_merge. - (cond3 OR cond4) didn't produce a usable range (e.g. they were t.key1part1=c1 AND t.key1part2=c1, respectively, and combining them didn't yield any range list) - All other scand didn't have their counterparts, so we'll end up with a SEL_TREE of: range(condM) AND index_merge((cond5 AND cond6),(cond7 AND cond8)) . O4. There is no O4. DISCARD-INDEX-MERGE-4 will remain there. The idea is that although DISCARD-INDEX-MERGE-4 does discard plans, so far we haven seen any complaints that could be attributed to it. If we face the need to lift DISCARD-INDEX-MERGE-4, our answer will be to lift it ,and produce a cross-product: ((key1p OR key2p) AND (key3p OR key4p)) OR ((key5p OR key6p) AND (key7p OR key8p)) = (key1p OR key2p OR key5p OR key6p) AND // this part is currently (key3p OR key4p OR key5p OR key6p) AND // produced (key1p OR key2p OR key5p OR key6p) AND // this part will be added (key3p OR key4p OR key5p OR key6p) //. In order to limit the impact of this combinatorial explosion, we will introduce a rule that we won't generate more than #defined MAX_IMERGE_OPTS options. 3. Testing and required coverage ================================ So far could find the following user cases: * BUG#17259: Query optimizer chooses wrong index * BUG#17673: Optimizer does not use Index Merge optimization in some cases * BUG#23322: Optimizer sometimes erroniously prefers other index over index merge * BUG#30151: optimizer is very reluctant to chose index_merge algorithm ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Psergey): Store in binlog text of statements that caused RBR events (47)
by worklog-noreply＠askmonty.org 15 Aug '09

15 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Store in binlog text of statements that caused RBR events CREATION DATE..: Sat, 15 Aug 2009, 23:48 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 47 (http://askmonty.org/worklog/?tid=47) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: DESCRIPTION: Store in binlog (and show in mysqlbinlog output) texts of statements that caused RBR events This is needed for (list from Monty): - Easier to understand why updates happened - Would make it easier to find out where in application things went wrong (as you can search for exact strings) - Allow one to filter things based on comments in the statement. The cost of this can be that the binlog will be approximately 2x in size (especially insert of big blob's would be a bit painful), so this should be an optional feature. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Psergey): Store in binlog text of statements that caused RBR events (47)
by worklog-noreply＠askmonty.org 15 Aug '09

15 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Store in binlog text of statements that caused RBR events CREATION DATE..: Sat, 15 Aug 2009, 23:48 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 47 (http://askmonty.org/worklog/?tid=47) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: DESCRIPTION: Store in binlog (and show in mysqlbinlog output) texts of statements that caused RBR events This is needed for (list from Monty): - Easier to understand why updates happened - Would make it easier to find out where in application things went wrong (as you can search for exact strings) - Allow one to filter things based on comments in the statement. The cost of this can be that the binlog will be approximately 2x in size (especially insert of big blob's would be a bit painful), so this should be an optional feature. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Change BINLOG statement syntax to be human-readable (46)
by worklog-noreply＠askmonty.org 15 Aug '09

15 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Change BINLOG statement syntax to be human-readable CREATION DATE..: Sat, 15 Aug 2009, 23:42 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 46 (http://askmonty.org/worklog/?tid=46) VERSION........: WorkLog-3.4 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Sat, 15 Aug 2009, 23:43)=-=- High-Level Specification modified. --- /tmp/wklog.46.old.17742 2009-08-15 23:43:09.000000000 +0300 +++ /tmp/wklog.46.new.17742 2009-08-15 23:43:09.000000000 +0300 @@ -1 +1,28 @@ +Suggestion 1 +------------ +Original syntax suggestion by Kristian: + + BINLOG + WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 + TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 + TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 + WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 + UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), + FROM ('a') TO ('b') FLAGS 0x0 + DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; + + This is basically a dump of what is stored in the events, and would be an + alternative to BINLOG 'gwWEShMBAA...'. + +Feedback and other suggestions +------------------------------ +* What is the need for WITH TIMESTAMP part? Can't one use a separate + SET TIMESTAMP statement? + +* mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something + that's close to readable SQL. Can we make it to be regular parseable SQL? + + This will be syntax that's familiar to our parser and to the users + - A stream of SQL statements will be slower to run than BINLOG statements + (due to locking, table open/close, etc). (TODO: is it really slower? we + haven't checked). DESCRIPTION: One of great things about mysqlbinlog was that its output was human-readable SQL, so it was possible to edit it manually or with help of scripts. With RBR events and BINLOG 'DpiGShMBAAAALQAAADcBAA...' statements this is no longer the case. This WL task is about making BINLOG statements to be human-readable (either as an option or by default HIGH-LEVEL SPECIFICATION: Suggestion 1 ------------ Original syntax suggestion by Kristian: BINLOG WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), FROM ('a') TO ('b') FLAGS 0x0 DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; This is basically a dump of what is stored in the events, and would be an alternative to BINLOG 'gwWEShMBAA...'. Feedback and other suggestions ------------------------------ * What is the need for WITH TIMESTAMP part? Can't one use a separate SET TIMESTAMP statement? * mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something that's close to readable SQL. Can we make it to be regular parseable SQL? + This will be syntax that's familiar to our parser and to the users - A stream of SQL statements will be slower to run than BINLOG statements (due to locking, table open/close, etc). (TODO: is it really slower? we haven't checked). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Change BINLOG statement syntax to be human-readable (46)
by worklog-noreply＠askmonty.org 15 Aug '09

15 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Change BINLOG statement syntax to be human-readable CREATION DATE..: Sat, 15 Aug 2009, 23:42 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 46 (http://askmonty.org/worklog/?tid=46) VERSION........: WorkLog-3.4 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Sat, 15 Aug 2009, 23:43)=-=- High-Level Specification modified. --- /tmp/wklog.46.old.17742 2009-08-15 23:43:09.000000000 +0300 +++ /tmp/wklog.46.new.17742 2009-08-15 23:43:09.000000000 +0300 @@ -1 +1,28 @@ +Suggestion 1 +------------ +Original syntax suggestion by Kristian: + + BINLOG + WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 + TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 + TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 + WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 + UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), + FROM ('a') TO ('b') FLAGS 0x0 + DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; + + This is basically a dump of what is stored in the events, and would be an + alternative to BINLOG 'gwWEShMBAA...'. + +Feedback and other suggestions +------------------------------ +* What is the need for WITH TIMESTAMP part? Can't one use a separate + SET TIMESTAMP statement? + +* mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something + that's close to readable SQL. Can we make it to be regular parseable SQL? + + This will be syntax that's familiar to our parser and to the users + - A stream of SQL statements will be slower to run than BINLOG statements + (due to locking, table open/close, etc). (TODO: is it really slower? we + haven't checked). DESCRIPTION: One of great things about mysqlbinlog was that its output was human-readable SQL, so it was possible to edit it manually or with help of scripts. With RBR events and BINLOG 'DpiGShMBAAAALQAAADcBAA...' statements this is no longer the case. This WL task is about making BINLOG statements to be human-readable (either as an option or by default HIGH-LEVEL SPECIFICATION: Suggestion 1 ------------ Original syntax suggestion by Kristian: BINLOG WITH TIMESTAMP xxx SERVER_ID 1 MASTER_POS 415 FLAGS 0x0 TABLE db1.table1 AS 1 COLUMNS (INT NOT NULL, BLOB, VARCHAR(100)) FLAGS 0x0 TABLE db2.table2 AS 2 COLUMNS (CHAR(10)) FLAGS 0x0 WRITE_ROW INTO db1.table1(1,3) VALUES (42, 'foobar'), (10, NULL) FLAGS 0x2 UPDATE_ROW INTO db2.table2 (1) (1) VALUES FROM ('beforeval') TO ('toval'), FROM ('a') TO ('b') FLAGS 0x0 DELETE_ROW INTO db2.table2 (1) VALUES ('row_to_delete') FLAGS 0x0; This is basically a dump of what is stored in the events, and would be an alternative to BINLOG 'gwWEShMBAA...'. Feedback and other suggestions ------------------------------ * What is the need for WITH TIMESTAMP part? Can't one use a separate SET TIMESTAMP statement? * mysqlbinlog --base64-output=DECODE-ROWS --verbose already produces something that's close to readable SQL. Can we make it to be regular parseable SQL? + This will be syntax that's familiar to our parser and to the users - A stream of SQL statements will be slower to run than BINLOG statements (due to locking, table open/close, etc). (TODO: is it really slower? we haven't checked). ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Psergey): Change BINLOG statement syntax to be human-readable (46)
by worklog-noreply＠askmonty.org 15 Aug '09

15 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Change BINLOG statement syntax to be human-readable CREATION DATE..: Sat, 15 Aug 2009, 23:42 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 46 (http://askmonty.org/worklog/?tid=46) VERSION........: WorkLog-3.4 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: DESCRIPTION: One of great things about mysqlbinlog was that its output was human-readable SQL, so it was possible to edit it manually or with help of scripts. With RBR events and BINLOG 'DpiGShMBAAAALQAAADcBAA...' statements this is no longer the case. This WL task is about making BINLOG statements to be human-readable (either as an option or by default ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Psergey): Change BINLOG statement syntax to be human-readable (46)
by worklog-noreply＠askmonty.org 15 Aug '09

15 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Change BINLOG statement syntax to be human-readable CREATION DATE..: Sat, 15 Aug 2009, 23:42 SUPERVISOR.....: Monty IMPLEMENTOR....: Psergey COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 46 (http://askmonty.org/worklog/?tid=46) VERSION........: WorkLog-3.4 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: DESCRIPTION: One of great things about mysqlbinlog was that its output was human-readable SQL, so it was possible to edit it manually or with help of scripts. With RBR events and BINLOG 'DpiGShMBAAAALQAAADcBAA...' statements this is no longer the case. This WL task is about making BINLOG statements to be human-readable (either as an option or by default ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Add a mysqlbinlog option to produce succint output (45)
by worklog-noreply＠askmonty.org 15 Aug '09

15 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add a mysqlbinlog option to produce succint output CREATION DATE..: Sat, 15 Aug 2009, 23:40 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 45 (http://askmonty.org/worklog/?tid=45) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Sat, 15 Aug 2009, 23:40)=-=- Title modified. --- /tmp/wklog.45.old.17603 2009-08-15 23:40:38.000000000 +0300 +++ /tmp/wklog.45.new.17603 2009-08-15 23:40:38.000000000 +0300 @@ -1 +1 @@ -Add a mysqlbinlog option to produce siccint output +Add a mysqlbinlog option to produce succint output DESCRIPTION: Add a mysqlbinlog option to produce the most succinct output, without any comments or other statements that are not needed to apply binlog correctly. This will be different from --short-form option. That option causes mysqlbinlog not to print RBR events, i.e. the output is not supposed to be applied. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Psergey): Add a mysqlbinlog option to produce succint output (45)
by worklog-noreply＠askmonty.org 15 Aug '09

15 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add a mysqlbinlog option to produce succint output CREATION DATE..: Sat, 15 Aug 2009, 23:40 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 45 (http://askmonty.org/worklog/?tid=45) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Psergey - Sat, 15 Aug 2009, 23:40)=-=- Title modified. --- /tmp/wklog.45.old.17603 2009-08-15 23:40:38.000000000 +0300 +++ /tmp/wklog.45.new.17603 2009-08-15 23:40:38.000000000 +0300 @@ -1 +1 @@ -Add a mysqlbinlog option to produce siccint output +Add a mysqlbinlog option to produce succint output DESCRIPTION: Add a mysqlbinlog option to produce the most succinct output, without any comments or other statements that are not needed to apply binlog correctly. This will be different from --short-form option. That option causes mysqlbinlog not to print RBR events, i.e. the output is not supposed to be applied. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Psergey): Add a mysqlbinlog option to produce siccint output (45)
by worklog-noreply＠askmonty.org 15 Aug '09

15 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add a mysqlbinlog option to produce siccint output CREATION DATE..: Sat, 15 Aug 2009, 23:40 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 45 (http://askmonty.org/worklog/?tid=45) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: DESCRIPTION: Add a mysqlbinlog option to produce the most succinct output, without any comments or other statements that are not needed to apply binlog correctly. This will be different from --short-form option. That option causes mysqlbinlog not to print RBR events, i.e. the output is not supposed to be applied. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] New (by Psergey): Add a mysqlbinlog option to produce siccint output (45)
by worklog-noreply＠askmonty.org 15 Aug '09

15 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add a mysqlbinlog option to produce siccint output CREATION DATE..: Sat, 15 Aug 2009, 23:40 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Server-RawIdeaBin TASK ID........: 45 (http://askmonty.org/worklog/?tid=45) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: DESCRIPTION: Add a mysqlbinlog option to produce the most succinct output, without any comments or other statements that are not needed to apply binlog correctly. This will be different from --short-form option. That option causes mysqlbinlog not to print RBR events, i.e. the output is not supposed to be applied. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Rev 2724: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r10/
by Sergey Petrunya 15 Aug '09

15 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r10/ ------------------------------------------------------------ revno: 2724 revision-id: psergey(a)askmonty.org-20090815153912-q47vfp1j22ilmup2 parent: psergey(a)askmonty.org-20090815121442-706m9ujn8km4u4y1 committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Sat 2009-08-15 18:39:12 +0300 message: MWL#17: Table elimination - Review feedback, more variable renames === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-15 12:14:42 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-15 15:39:12 +0000 @@ -114,7 +114,6 @@ MODULE_EXPRESSION, MODULE_MULTI_EQUALITY, MODULE_UNIQUE_KEY, - MODULE_TABLE, MODULE_OUTER_JOIN } type; /* Type of the object */ @@ -138,7 +137,7 @@ { public: Field_value *field; - Item *val; + Item *expression; /* Used during condition analysis only, similar to KEYUSE::level */ uint level; @@ -510,18 +509,18 @@ */ if (old->field == new_fields->field) { - if (!new_fields->val->const_item()) + if (!new_fields->expression->const_item()) { /* If the value matches, we can use the key reference. If not, we keep it until we have examined all new values */ - if (old->val->eq(new_fields->val, old->field->field->binary())) + if (old->expression->eq(new_fields->expression, old->field->field->binary())) { old->level= and_level; } } - else if (old->val->eq_by_collation(new_fields->val, + else if (old->expression->eq_by_collation(new_fields->expression, old->field->field->binary(), old->field->field->charset())) { @@ -633,7 +632,7 @@ /* Store possible eq field */ (*eq_dep)->type= Module_dep::MODULE_EXPRESSION; //psergey-todo; (*eq_dep)->field= get_field_value(te, field); - (*eq_dep)->val= *value; + (*eq_dep)->expression= *value; (*eq_dep)->level= and_level; (*eq_dep)++; } @@ -953,7 +952,7 @@ { deps_setter.expr_offset= eq_dep - te->equality_deps; eq_dep->unknown_args= 0; - eq_dep->val->walk(&Item::check_column_usage_processor, FALSE, + eq_dep->expression->walk(&Item::check_column_usage_processor, FALSE, (uchar*)&deps_setter); if (!eq_dep->unknown_args) { @@ -1283,7 +1282,7 @@ char buf[128]; String str(buf, sizeof(buf), &my_charset_bin); str.length(0); - eq_dep->val->print(&str, QT_ORDINARY); + eq_dep->expression->print(&str, QT_ORDINARY); fprintf(DBUG_FILE, " equality%d: %s -> %s.%s\n", eq_dep - te->equality_deps, str.c_ptr(),

1 0

[Maria-developers] Rev 2723: Fix trivial typo in file:///home/psergey/dev/maria-5.1-table-elim-r10/
by Sergey Petrunya 15 Aug '09

15 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r10/ ------------------------------------------------------------ revno: 2723 revision-id: psergey(a)askmonty.org-20090815121442-706m9ujn8km4u4y1 parent: psergey(a)askmonty.org-20090815102953-7s0jb470ibwq58qz committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Sat 2009-08-15 16:14:42 +0400 message: Fix trivial typo === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-15 10:29:53 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-15 12:14:42 +0000 @@ -841,7 +841,7 @@ *eliminable_tables); } - if (eliminable && get_outer_join_dep(te, tbl, cur_map)) + if (eliminable && !get_outer_join_dep(te, tbl, cur_map)) return TRUE; tables_used_on_left |= tbl->on_expr->used_tables();

1 0

[Maria-developers] Rev 2722: MWL#17: Table elimination in file:///home/psergey/dev/maria-5.1-table-elim-r10/
by Sergey Petrunya 15 Aug '09

15 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r10/ ------------------------------------------------------------ revno: 2722 revision-id: psergey(a)askmonty.org-20090815102953-7s0jb470ibwq58qz parent: psergey(a)askmonty.org-20090815060803-0yvp5mmgo87emykp committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Sat 2009-08-15 14:29:53 +0400 message: MWL#17: Table elimination Continue with addressing review feedback part two: - rename enum members - add checking for out of memory errors on allocation === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-15 06:08:03 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-15 10:29:53 +0000 @@ -111,11 +111,11 @@ { public: enum { - FD_EXPRESSION, - FD_MULTI_EQUALITY, - FD_UNIQUE_KEY, - FD_TABLE, - FD_OUTER_JOIN + MODULE_EXPRESSION, + MODULE_MULTI_EQUALITY, + MODULE_UNIQUE_KEY, + MODULE_TABLE, + MODULE_OUTER_JOIN } type; /* Type of the object */ /* @@ -156,7 +156,7 @@ Key_module(Table_value *table_arg, uint keyno_arg, uint n_parts_arg) : table(table_arg), keyno(keyno_arg), next_table_key(NULL) { - type= Module_dep::FD_UNIQUE_KEY; + type= Module_dep::MODULE_UNIQUE_KEY; unknown_args= n_parts_arg; } Table_value *table; /* Table this key is from */ @@ -178,7 +178,7 @@ Outer_join_module(TABLE_LIST *table_list_arg, uint n_children) : table_list(table_list_arg), parent(NULL) { - type= Module_dep::FD_OUTER_JOIN; + type= Module_dep::MODULE_OUTER_JOIN; unknown_args= n_children; } /* @@ -205,7 +205,7 @@ class Table_elimination { public: - Table_elimination(JOIN *join_arg) : join(join_arg) + Table_elimination(JOIN *join_arg) : join(join_arg), n_outer_joins(0) { bzero(table_deps, sizeof(table_deps)); } @@ -220,6 +220,7 @@ /* Outer joins that are candidates for elimination */ List<Outer_join_module> oj_deps; + uint n_outer_joins; /* Bitmap of how expressions depend on bits */ MY_BITMAP expr_deps; @@ -630,22 +631,25 @@ DBUG_ASSERT(eq_func); /* Store possible eq field */ - (*eq_dep)->type= Module_dep::FD_EXPRESSION; //psergey-todo; + (*eq_dep)->type= Module_dep::MODULE_EXPRESSION; //psergey-todo; (*eq_dep)->field= get_field_value(te, field); (*eq_dep)->val= *value; (*eq_dep)->level= and_level; (*eq_dep)++; } + /* Get a Table_value object for the given table, creating it if necessary. */ static Table_value *get_table_value(Table_elimination *te, TABLE *table) { - Table_value *tbl_dep= new Table_value(table); + Table_value *tbl_dep; + if (!(tbl_dep= new Table_value(table))) + return NULL; + Key_module **key_list= &(tbl_dep->keys); - /* Add dependencies for unique keys */ for (uint i=0; i < table->s->keys; i++) { @@ -657,7 +661,7 @@ key_list= &(key_dep->next_table_key); } } - return te->table_deps[table->tablenr] = tbl_dep; + return te->table_deps[table->tablenr]= tbl_dep; } @@ -672,7 +676,10 @@ /* First, get the table*/ if (!(tbl_dep= te->table_deps[table->tablenr])) - tbl_dep= get_table_value(te, table); + { + if (!(tbl_dep= get_table_value(te, table))) + return NULL; + } /* Try finding the field in field list */ Field_value **pfield= &(tbl_dep->fields); @@ -702,10 +709,12 @@ static Outer_join_module *get_outer_join_dep(Table_elimination *te, - TABLE_LIST *outer_join, table_map deps_map) + TABLE_LIST *outer_join, + table_map deps_map) { Outer_join_module *oj_dep; oj_dep= new Outer_join_module(outer_join, my_count_bits(deps_map)); + te->n_outer_joins++; /* Collect a bitmap fo tables that we depend on, and also set parent pointer @@ -734,7 +743,8 @@ } } DBUG_ASSERT(table); - table_dep= get_table_value(te, table); + if (!(table_dep= get_table_value(te, table))) + return NULL; } /* @@ -781,7 +791,7 @@ . */ -static void +static bool collect_funcdeps_for_join_list(Table_elimination *te, List<TABLE_LIST> *join_list, bool build_eq_deps, @@ -808,11 +818,12 @@ eliminable= !(cur_map & outside_used_tables); if (eliminable) *eliminable_tables |= cur_map; - collect_funcdeps_for_join_list(te, &tbl->nested_join->join_list, - eliminable || build_eq_deps, - outside_used_tables, - eliminable_tables, - eq_dep); + if (collect_funcdeps_for_join_list(te, &tbl->nested_join->join_list, + eliminable || build_eq_deps, + outside_used_tables, + eliminable_tables, + eq_dep)) + return TRUE; } else { @@ -830,13 +841,13 @@ *eliminable_tables); } - if (eliminable) - te->oj_deps.push_back(get_outer_join_dep(te, tbl, cur_map)); + if (eliminable && get_outer_join_dep(te, tbl, cur_map)) + return TRUE; tables_used_on_left |= tbl->on_expr->used_tables(); } } - return; + return FALSE; } @@ -1053,16 +1064,18 @@ DBUG_VOID_RETURN; Equality_module *eq_deps_end= te.equality_deps; table_map eliminable_tables= 0; - collect_funcdeps_for_join_list(&te, join->join_list, - FALSE, - used_tables, - &eliminable_tables, - &eq_deps_end); + if (collect_funcdeps_for_join_list(&te, join->join_list, + FALSE, + used_tables, + &eliminable_tables, + &eq_deps_end)) + DBUG_VOID_RETURN; te.n_equality_deps= eq_deps_end - te.equality_deps; Module_dep *bound_modules; //Value_dep *bound_values; - setup_equality_deps(&te, &bound_modules); + if (setup_equality_deps(&te, &bound_modules)) + DBUG_VOID_RETURN; run_elimination_wave(&te, bound_modules); } @@ -1108,7 +1121,7 @@ { switch (bound_modules->type) { - case Module_dep::FD_EXPRESSION: + case Module_dep::MODULE_EXPRESSION: { /* It's a field=expr and we got to know the expr, so we know the field */ Equality_module *eq_dep= (Equality_module*)bound_modules; @@ -1121,7 +1134,7 @@ } break; } - case Module_dep::FD_UNIQUE_KEY: + case Module_dep::MODULE_UNIQUE_KEY: { /* Unique key is known means the table is known */ Table_value *table_dep=((Key_module*)bound_modules)->table; @@ -1134,13 +1147,13 @@ } break; } - case Module_dep::FD_OUTER_JOIN: + case Module_dep::MODULE_OUTER_JOIN: { Outer_join_module *outer_join_dep= (Outer_join_module*)bound_modules; mark_as_eliminated(te->join, outer_join_dep->table_list); break; } - case Module_dep::FD_MULTI_EQUALITY: + case Module_dep::MODULE_MULTI_EQUALITY: default: DBUG_ASSERT(0); }

1 0

[Maria-developers] Rev 2721: MWL#17: Address 2nd post-review feedback in file:///home/psergey/dev/maria-5.1-table-elim-r10/
by Sergey Petrunya 15 Aug '09

15 Aug '09

At file:///home/psergey/dev/maria-5.1-table-elim-r10/ ------------------------------------------------------------ revno: 2721 revision-id: psergey(a)askmonty.org-20090815060803-0yvp5mmgo87emykp parent: psergey(a)askmonty.org-20090813211212-jghejwxsl6adtopl committer: Sergey Petrunya <psergey(a)askmonty.org> branch nick: maria-5.1-table-elim-r10 timestamp: Sat 2009-08-15 10:08:03 +0400 message: MWL#17: Address 2nd post-review feedback - Switch from uniform graph to bipartite graph with two kinds of nodes: "values" (tables and fields) and "modules" (t.col=func(...) equalities, multi-equalities, unique keys, inner sides of outer joins). - Rename functions, classes, etc. === modified file 'sql/opt_table_elimination.cc' --- a/sql/opt_table_elimination.cc 2009-08-13 20:44:52 +0000 +++ b/sql/opt_table_elimination.cc 2009-08-15 06:08:03 +0000 @@ -40,19 +40,78 @@ Table elimination is redone on every PS re-execution. */ - -/* - An abstract structure that represents some entity that's being dependent on - some other entity. -*/ - -class Func_dep : public Sql_alloc -{ -public: - enum { - FD_INVALID, +class Value_dep +{ +public: + enum { + VALUE_FIELD, + VALUE_TABLE, + } type; /* Type of the object */ + + bool bound; + Value_dep *next; +}; + +class Field_value; +class Table_value; +class Outer_join_module; +class Key_module; + +/* + A table field. There is only one such object for any tblX.fieldY + - the field epends on its table and equalities + - expressions that use the field are its dependencies +*/ +class Field_value : public Value_dep +{ +public: + Field_value(Table_value *table_arg, Field *field_arg) : + table(table_arg), field(field_arg) + { + type= Value_dep::VALUE_FIELD; + } + + Table_value *table; /* Table this field is from */ + Field *field; + + /* + Field_deps that belong to one table form a linked list. list members are + ordered by field_index + */ + Field_value *next_table_field; + uint bitmap_offset; /* Offset of our part of the bitmap */ +}; + + +/* + A table. + - table depends on any of its unique keys + - has its fields and embedding outer join as dependency. +*/ +class Table_value : public Value_dep +{ +public: + Table_value(TABLE *table_arg) : + table(table_arg), fields(NULL), keys(NULL), outer_join_dep(NULL) + { + type= Value_dep::VALUE_TABLE; + } + TABLE *table; + Field_value *fields; /* Ordered list of fields that belong to this table */ + Key_module *keys; /* Ordered list of Unique keys in this table */ + Outer_join_module *outer_join_dep; /* Innermost eliminable outer join we're in */ +}; + + +/* + A 'module' +*/ + +class Module_dep : public Sql_alloc +{ +public: + enum { FD_EXPRESSION, - FD_FIELD, FD_MULTI_EQUALITY, FD_UNIQUE_KEY, FD_TABLE, @@ -63,58 +122,26 @@ Used to make a linked list of elements that became bound and thus can make elements that depend on them bound, too. */ - Func_dep *next; - bool bound; /* TRUE<=> The entity is considered bound */ - Func_dep() : next(NULL), bound(FALSE) {} + Module_dep *next; + uint unknown_args; /* TRUE<=> The entity is considered bound */ + + Module_dep() : next(NULL), unknown_args(0) {} }; -class Field_dep; -class Table_dep; -class Outer_join_dep; - /* A "tbl.column= expr" equality dependency. tbl.column depends on fields used in expr. */ -class Equality_dep : public Func_dep +class Equality_module : public Module_dep { public: - Field_dep *field; + Field_value *field; Item *val; /* Used during condition analysis only, similar to KEYUSE::level */ uint level; - - /* Number of fields referenced from *val that are not yet 'bound' */ - uint unknown_args; -}; - - -/* - A table field. There is only one such object for any tblX.fieldY - - the field epends on its table and equalities - - expressions that use the field are its dependencies -*/ -class Field_dep : public Func_dep -{ -public: - Field_dep(Table_dep *table_arg, Field *field_arg) : - table(table_arg), field(field_arg) - { - type= Func_dep::FD_FIELD; - } - - Table_dep *table; /* Table this field is from */ - Field *field; - - /* - Field_deps that belong to one table form a linked list. list members are - ordered by field_index - */ - Field_dep *next_table_field; - uint bitmap_offset; /* Offset of our part of the bitmap */ }; @@ -123,41 +150,21 @@ - Unique key depends on all of its components - Key's table is its dependency */ -class Key_dep: public Func_dep +class Key_module: public Module_dep { public: - Key_dep(Table_dep *table_arg, uint keyno_arg, uint n_parts_arg) : - table(table_arg), keyno(keyno_arg), n_missing_keyparts(n_parts_arg), - next_table_key(NULL) + Key_module(Table_value *table_arg, uint keyno_arg, uint n_parts_arg) : + table(table_arg), keyno(keyno_arg), next_table_key(NULL) { - type= Func_dep::FD_UNIQUE_KEY; + type= Module_dep::FD_UNIQUE_KEY; + unknown_args= n_parts_arg; } - Table_dep *table; /* Table this key is from */ + Table_value *table; /* Table this key is from */ uint keyno; - uint n_missing_keyparts; /* Unique keys form a linked list, ordered by keyno */ - Key_dep *next_table_key; -}; - - -/* - A table. - - table depends on any of its unique keys - - has its fields and embedding outer join as dependency. -*/ -class Table_dep : public Func_dep -{ -public: - Table_dep(TABLE *table_arg) : - table(table_arg), fields(NULL), keys(NULL), outer_join_dep(NULL) - { - type= Func_dep::FD_TABLE; - } - TABLE *table; - Field_dep *fields; /* Ordered list of fields that belong to this table */ - Key_dep *keys; /* Ordered list of Unique keys in this table */ - Outer_join_dep *outer_join_dep; /* Innermost eliminable outer join we're in */ -}; + Key_module *next_table_key; +}; + /* @@ -165,14 +172,14 @@ - it depends on all tables inside it - has its parent outer join as dependency */ -class Outer_join_dep: public Func_dep +class Outer_join_module: public Module_dep { public: - Outer_join_dep(TABLE_LIST *table_list_arg, table_map missing_tables_arg) : - table_list(table_list_arg), missing_tables(missing_tables_arg), - all_tables(missing_tables_arg), parent(NULL) + Outer_join_module(TABLE_LIST *table_list_arg, uint n_children) : + table_list(table_list_arg), parent(NULL) { - type= Func_dep::FD_OUTER_JOIN; + type= Module_dep::FD_OUTER_JOIN; + unknown_args= n_children; } /* Outer join we're representing. This can be a join nest or a one table that @@ -184,11 +191,11 @@ Tables within this outer join (and its descendants) that are not yet known to be functionally dependent. */ - table_map missing_tables; + table_map missing_tables; //psergey-todo: remove /* All tables within this outer join and its descendants */ - table_map all_tables; + table_map all_tables; //psergey-todo: remove /* Parent eliminable outer join, if any */ - Outer_join_dep *parent; + Outer_join_module *parent; }; @@ -205,44 +212,45 @@ JOIN *join; /* Array of equality dependencies */ - Equality_dep *equality_deps; + Equality_module *equality_deps; uint n_equality_deps; /* Number of elements in the array */ - /* tablenr -> Table_dep* mapping. */ - Table_dep *table_deps[MAX_KEY]; + /* tablenr -> Table_value* mapping. */ + Table_value *table_deps[MAX_KEY]; /* Outer joins that are candidates for elimination */ - List<Outer_join_dep> oj_deps; + List<Outer_join_module> oj_deps; /* Bitmap of how expressions depend on bits */ MY_BITMAP expr_deps; }; - static -void build_eq_deps_for_cond(Table_elimination *te, Equality_dep **fdeps, +void build_eq_deps_for_cond(Table_elimination *te, Equality_module **fdeps, uint *and_level, Item *cond, table_map usable_tables); static void add_eq_dep(Table_elimination *te, - Equality_dep **eq_dep, uint and_level, + Equality_module **eq_dep, uint and_level, Item_func *cond, Field *field, bool eq_func, Item **value, uint num_values, table_map usable_tables); static -Equality_dep *merge_func_deps(Equality_dep *start, Equality_dep *new_fields, - Equality_dep *end, uint and_level); - -static Table_dep *get_table_dep(Table_elimination *te, TABLE *table); -static Field_dep *get_field_dep(Table_elimination *te, Field *field); - +Equality_module *merge_func_deps(Equality_module *start, Equality_module *new_fields, + Equality_module *end, uint and_level); + +static Table_value *get_table_value(Table_elimination *te, TABLE *table); +static Field_value *get_field_value(Table_elimination *te, Field *field); +static +void run_elimination_wave(Table_elimination *te, Module_dep *bound_modules); void eliminate_tables(JOIN *join); static void mark_as_eliminated(JOIN *join, TABLE_LIST *tbl); +#if 0 #ifndef DBUG_OFF static void dbug_print_deps(Table_elimination *te); #endif - +#endif /*******************************************************************************************/ /* @@ -262,14 +270,14 @@ */ static -void build_eq_deps_for_cond(Table_elimination *te, Equality_dep **fdeps, +void build_eq_deps_for_cond(Table_elimination *te, Equality_module **fdeps, uint *and_level, Item *cond, table_map usable_tables) { if (cond->type() == Item_func::COND_ITEM) { List_iterator_fast<Item> li(*((Item_cond*) cond)->argument_list()); - Equality_dep *org_key_fields= *fdeps; + Equality_module *org_key_fields= *fdeps; /* AND/OR */ if (((Item_cond*) cond)->functype() == Item_func::COND_AND_FUNC) @@ -293,7 +301,7 @@ Item *item; while ((item=li++)) { - Equality_dep *start_key_fields= *fdeps; + Equality_module *start_key_fields= *fdeps; (*and_level)++; build_eq_deps_for_cond(te, fdeps, and_level, item, usable_tables); *fdeps= merge_func_deps(org_key_fields, start_key_fields, *fdeps, @@ -432,7 +440,7 @@ /* - Perform an OR operation on two (adjacent) Equality_dep arrays. + Perform an OR operation on two (adjacent) Equality_module arrays. SYNOPSIS merge_func_deps() @@ -442,7 +450,7 @@ and_level AND-level. DESCRIPTION - This function is invoked for two adjacent arrays of Equality_dep elements: + This function is invoked for two adjacent arrays of Equality_module elements: $LEFT_PART $RIGHT_PART +-----------------------+-----------------------+ @@ -477,19 +485,19 @@ */ static -Equality_dep *merge_func_deps(Equality_dep *start, Equality_dep *new_fields, - Equality_dep *end, uint and_level) +Equality_module *merge_func_deps(Equality_module *start, Equality_module *new_fields, + Equality_module *end, uint and_level) { if (start == new_fields) return start; // Impossible or if (new_fields == end) return start; // No new fields, skip all - Equality_dep *first_free=new_fields; + Equality_module *first_free=new_fields; for (; new_fields != end ; new_fields++) { - for (Equality_dep *old=start ; old != first_free ; old++) + for (Equality_module *old=start ; old != first_free ; old++) { /* TODO: does it make sense to attempt to merging multiple-equalities? @@ -534,7 +542,7 @@ Ok, the results are within the [start, first_free) range, and the useful elements have level==and_level. Now, lets remove all unusable elements: */ - for (Equality_dep *old=start ; old != first_free ;) + for (Equality_module *old=start ; old != first_free ;) { if (old->level != and_level) { // Not used in all levels @@ -550,14 +558,14 @@ /* - Add an Equality_dep element for a given predicate, if applicable + Add an Equality_module element for a given predicate, if applicable DESCRIPTION This function is modeled after add_key_field(). */ static -void add_eq_dep(Table_elimination *te, Equality_dep **eq_dep, +void add_eq_dep(Table_elimination *te, Equality_module **eq_dep, uint and_level, Item_func *cond, Field *field, bool eq_func, Item **value, uint num_values, table_map usable_tables) @@ -622,22 +630,21 @@ DBUG_ASSERT(eq_func); /* Store possible eq field */ - (*eq_dep)->type= Func_dep::FD_EXPRESSION; //psergey-todo; - (*eq_dep)->field= get_field_dep(te, field); + (*eq_dep)->type= Module_dep::FD_EXPRESSION; //psergey-todo; + (*eq_dep)->field= get_field_value(te, field); (*eq_dep)->val= *value; (*eq_dep)->level= and_level; (*eq_dep)++; } - /* - Get a Table_dep object for the given table, creating it if necessary. + Get a Table_value object for the given table, creating it if necessary. */ -static Table_dep *get_table_dep(Table_elimination *te, TABLE *table) +static Table_value *get_table_value(Table_elimination *te, TABLE *table) { - Table_dep *tbl_dep= new Table_dep(table); - Key_dep **key_list= &(tbl_dep->keys); + Table_value *tbl_dep= new Table_value(table); + Key_module **key_list= &(tbl_dep->keys); /* Add dependencies for unique keys */ for (uint i=0; i < table->s->keys; i++) @@ -645,7 +652,7 @@ KEY *key= table->key_info + i; if ((key->flags & (HA_NOSAME | HA_END_SPACE_KEY)) == HA_NOSAME) { - Key_dep *key_dep= new Key_dep(tbl_dep, i, key->key_parts); + Key_module *key_dep= new Key_module(tbl_dep, i, key->key_parts); *key_list= key_dep; key_list= &(key_dep->next_table_key); } @@ -655,20 +662,20 @@ /* - Get a Field_dep object for the given field, creating it if necessary + Get a Field_value object for the given field, creating it if necessary */ -static Field_dep *get_field_dep(Table_elimination *te, Field *field) +static Field_value *get_field_value(Table_elimination *te, Field *field) { TABLE *table= field->table; - Table_dep *tbl_dep; + Table_value *tbl_dep; /* First, get the table*/ if (!(tbl_dep= te->table_deps[table->tablenr])) - tbl_dep= get_table_dep(te, table); + tbl_dep= get_table_value(te, table); /* Try finding the field in field list */ - Field_dep **pfield= &(tbl_dep->fields); + Field_value **pfield= &(tbl_dep->fields); while (*pfield && (*pfield)->field->field_index < field->field_index) { pfield= &((*pfield)->next_table_field); @@ -677,7 +684,7 @@ return *pfield; /* Create the field and insert it in the list */ - Field_dep *new_field= new Field_dep(tbl_dep, field); + Field_value *new_field= new Field_value(tbl_dep, field); new_field->next_table_field= *pfield; *pfield= new_field; @@ -686,19 +693,19 @@ /* - Create an Outer_join_dep object for the given outer join + Create an Outer_join_module object for the given outer join DESCRIPTION - Outer_join_dep objects for children (or further descendants) are always + Outer_join_module objects for children (or further descendants) are always created before the parents. */ static -Outer_join_dep *get_outer_join_dep(Table_elimination *te, +Outer_join_module *get_outer_join_dep(Table_elimination *te, TABLE_LIST *outer_join, table_map deps_map) { - Outer_join_dep *oj_dep; - oj_dep= new Outer_join_dep(outer_join, deps_map); + Outer_join_module *oj_dep; + oj_dep= new Outer_join_module(outer_join, my_count_bits(deps_map)); /* Collect a bitmap fo tables that we depend on, and also set parent pointer @@ -708,7 +715,7 @@ int idx; while ((idx= it.next_bit()) != Table_map_iterator::BITMAP_END) { - Table_dep *table_dep; + Table_value *table_dep; if (!(table_dep= te->table_deps[idx])) { /* @@ -727,23 +734,24 @@ } } DBUG_ASSERT(table); - table_dep= get_table_dep(te, table); + table_dep= get_table_value(te, table); } /* Walk from the table up to its embedding outer joins. The goal is to find the least embedded outer join nest and set its parent pointer to - point to the newly created Outer_join_dep. + point to the newly created Outer_join_module. to set the pointer of its near */ if (!table_dep->outer_join_dep) table_dep->outer_join_dep= oj_dep; else { - Outer_join_dep *oj= table_dep->outer_join_dep; + Outer_join_module *oj= table_dep->outer_join_dep; while (oj->parent) oj= oj->parent; - oj->parent=oj_dep; + if (oj != oj_dep) + oj->parent=oj_dep; } } return oj_dep; @@ -757,7 +765,7 @@ collect_funcdeps_for_join_list() te Table elimination context. join_list Join list to work on - build_eq_deps TRUE <=> build Equality_dep elements for all + build_eq_deps TRUE <=> build Equality_module elements for all members of the join list, even if they cannot be individually eliminated tables_used_elsewhere Bitmap of tables that are referred to from @@ -779,7 +787,7 @@ bool build_eq_deps, table_map tables_used_elsewhere, table_map *eliminable_tables, - Equality_dep **eq_dep) + Equality_module **eq_dep) { TABLE_LIST *tbl; List_iterator<TABLE_LIST> it(*join_list); @@ -845,10 +853,10 @@ void see_field(Field *field) { - Table_dep *tbl_dep; + Table_value *tbl_dep; if ((tbl_dep= te->table_deps[field->table->tablenr])) { - for (Field_dep *field_dep= tbl_dep->fields; field_dep; + for (Field_value *field_dep= tbl_dep->fields; field_dep; field_dep= field_dep->next_table_field) { if (field->field_index == field_dep->field->field_index) @@ -888,21 +896,21 @@ */ static -bool setup_equality_deps(Table_elimination *te, Func_dep **bound_deps_list) +bool setup_equality_deps(Table_elimination *te, Module_dep **bound_deps_list) { DBUG_ENTER("setup_equality_deps"); /* - Count Field_dep objects and assign each of them a unique bitmap_offset. + Count Field_value objects and assign each of them a unique bitmap_offset. */ uint offset= 0; - for (Table_dep **tbl_dep=te->table_deps; + for (Table_value **tbl_dep=te->table_deps; tbl_dep < te->table_deps + MAX_TABLES; tbl_dep++) { if (*tbl_dep) { - for (Field_dep *field_dep= (*tbl_dep)->fields; + for (Field_value *field_dep= (*tbl_dep)->fields; field_dep; field_dep= field_dep->next_table_field) { @@ -926,9 +934,9 @@ Also collect a linked list of equalities that are bound. */ - Func_dep *bound_dep= NULL; + Module_dep *bound_dep= NULL; Field_dependency_setter deps_setter(te); - for (Equality_dep *eq_dep= te->equality_deps; + for (Equality_module *eq_dep= te->equality_deps; eq_dep < te->equality_deps + te->n_equality_deps; eq_dep++) { @@ -940,12 +948,11 @@ { eq_dep->next= bound_dep; bound_dep= eq_dep; - eq_dep->bound= TRUE; } } *bound_deps_list= bound_dep; - DBUG_EXECUTE("test", dbug_print_deps(te); ); + //DBUG_EXECUTE("test", dbug_print_deps(te); ); DBUG_RETURN(FALSE); } @@ -1042,9 +1049,9 @@ uint m= max(thd->lex->current_select->max_equal_elems,1); uint max_elems= ((thd->lex->current_select->cond_count+1)*2 + thd->lex->current_select->between_count)*m + 1 + 10; - if (!(te.equality_deps= new Equality_dep[max_elems])) + if (!(te.equality_deps= new Equality_module[max_elems])) DBUG_VOID_RETURN; - Equality_dep *eq_deps_end= te.equality_deps; + Equality_module *eq_deps_end= te.equality_deps; table_map eliminable_tables= 0; collect_funcdeps_for_join_list(&te, join->join_list, FALSE, @@ -1052,96 +1059,125 @@ &eliminable_tables, &eq_deps_end); te.n_equality_deps= eq_deps_end - te.equality_deps; - Func_dep *bound_dep; - setup_equality_deps(&te, &bound_dep); - - /* - Run the wave. - All Func_dep-derived objects are divided into three classes: - - Those that have bound=FALSE - - Those that have bound=TRUE - - Those that have bound=TRUE and are in the list.. - - */ - while (bound_dep) - { - Func_dep *next= bound_dep->next; - //e= list.remove_first(); - switch (bound_dep->type) + + Module_dep *bound_modules; + //Value_dep *bound_values; + setup_equality_deps(&te, &bound_modules); + + run_elimination_wave(&te, bound_modules); + } + DBUG_VOID_RETURN; +} + + +static +void signal_from_field_to_exprs(Table_elimination* te, Field_value *field_dep, + Module_dep **bound_modules) +{ + /* Now, expressions */ + for (uint i=0; i < te->n_equality_deps; i++) + { + if (bitmap_is_set(&te->expr_deps, field_dep->bitmap_offset + i) && + te->equality_deps[i].unknown_args && + !--te->equality_deps[i].unknown_args) + { + /* Mark as bound and add to the list */ + Equality_module* eq_dep= &te->equality_deps[i]; + eq_dep->next= *bound_modules; + *bound_modules= eq_dep; + } + } +} + + +static +void run_elimination_wave(Table_elimination *te, Module_dep *bound_modules) +{ + Value_dep *bound_values= NULL; + /* + Run the wave. + All Func_dep-derived objects are divided into three classes: + - Those that have bound=FALSE + - Those that have bound=TRUE + - Those that have bound=TRUE and are in the list.. + + */ + while (bound_modules) + { + for (;bound_modules; bound_modules= bound_modules->next) + { + switch (bound_modules->type) { - case Func_dep::FD_EXPRESSION: + case Module_dep::FD_EXPRESSION: { /* It's a field=expr and we got to know the expr, so we know the field */ - Equality_dep *eq_dep= (Equality_dep*)bound_dep; + Equality_module *eq_dep= (Equality_module*)bound_modules; if (!eq_dep->field->bound) { /* Mark as bound and add to the list */ eq_dep->field->bound= TRUE; - eq_dep->field->next= next; - next= eq_dep->field; - } - break; - } - case Func_dep::FD_FIELD: + eq_dep->field->next= bound_values; + bound_values= eq_dep->field; + } + break; + } + case Module_dep::FD_UNIQUE_KEY: + { + /* Unique key is known means the table is known */ + Table_value *table_dep=((Key_module*)bound_modules)->table; + if (!table_dep->bound) + { + /* Mark as bound and add to the list */ + table_dep->bound= TRUE; + table_dep->next= bound_values; + bound_values= table_dep; + } + break; + } + case Module_dep::FD_OUTER_JOIN: + { + Outer_join_module *outer_join_dep= (Outer_join_module*)bound_modules; + mark_as_eliminated(te->join, outer_join_dep->table_list); + break; + } + case Module_dep::FD_MULTI_EQUALITY: + default: + DBUG_ASSERT(0); + } + } + + for (;bound_values; bound_values=bound_values->next) + { + switch (bound_values->type) + { + case Value_dep::VALUE_FIELD: { /* Field became known. Check out - unique keys we belong to - expressions that depend on us. */ - Field_dep *field_dep= (Field_dep*)bound_dep; - for (Key_dep *key_dep= field_dep->table->keys; key_dep; + Field_value *field_dep= (Field_value*)bound_values; + for (Key_module *key_dep= field_dep->table->keys; key_dep; key_dep= key_dep->next_table_key) { DBUG_PRINT("info", ("key %s.%s is now bound", key_dep->table->table->alias, key_dep->table->table->key_info[key_dep->keyno].name)); if (field_dep->field->part_of_key.is_set(key_dep->keyno) && - !key_dep->bound) - { - if (!--key_dep->n_missing_keyparts) - { - /* Mark as bound and add to the list */ - key_dep->bound= TRUE; - key_dep->next= next; - next= key_dep; - } - } - } - - /* Now, expressions */ - for (uint i=0; i < te.n_equality_deps; i++) - { - if (bitmap_is_set(&te.expr_deps, field_dep->bitmap_offset + i)) - { - Equality_dep* eq_dep= &te.equality_deps[i]; - if (!--eq_dep->unknown_args) - { - /* Mark as bound and add to the list */ - eq_dep->bound= TRUE; - eq_dep->next= next; - next= eq_dep; - } - } - } - break; - } - case Func_dep::FD_UNIQUE_KEY: - { - /* Unique key is known means the table is known */ - Table_dep *table_dep=((Key_dep*)bound_dep)->table; - if (!table_dep->bound) - { - /* Mark as bound and add to the list */ - table_dep->bound= TRUE; - table_dep->next= next; - next= table_dep; - } - break; - } - case Func_dep::FD_TABLE: - { - Table_dep *table_dep=(Table_dep*)bound_dep; + key_dep->unknown_args && !--key_dep->unknown_args) + { + /* Mark as bound and add to the list */ + key_dep->next= bound_modules; + bound_modules= key_dep; + } + } + signal_from_field_to_exprs(te, field_dep, &bound_modules); + break; + } + case Value_dep::VALUE_TABLE: + { + Table_value *table_dep=(Table_value*)bound_values; DBUG_PRINT("info", ("table %s is now bound", table_dep->table->alias)); /* @@ -1149,50 +1185,35 @@ - all its fields are known - one more element in outer join nest is known */ - for (Field_dep *field_dep= table_dep->fields; field_dep; + for (Field_value *field_dep= table_dep->fields; field_dep; field_dep= field_dep->next_table_field) { if (!field_dep->bound) { /* Mark as bound and add to the list */ field_dep->bound= TRUE; - field_dep->next= next; - next= field_dep; - } - } - Outer_join_dep *outer_join_dep= table_dep->outer_join_dep; - if (!(outer_join_dep->missing_tables &= ~table_dep->table->map)) - { - /* Mark as bound and add to the list */ - outer_join_dep->bound= TRUE; - outer_join_dep->next= next; - next= outer_join_dep; - } - break; - } - case Func_dep::FD_OUTER_JOIN: - { - Outer_join_dep *outer_join_dep= (Outer_join_dep*)bound_dep; - mark_as_eliminated(te.join, outer_join_dep->table_list); - Outer_join_dep *parent= outer_join_dep->parent; - if (parent && - !(parent->missing_tables &= ~outer_join_dep->all_tables)) - { - /* Mark as bound and add to the list */ - parent->bound= TRUE; - parent->next= next; - next= parent; - } - break; - } - case Func_dep::FD_MULTI_EQUALITY: - default: + signal_from_field_to_exprs(te, field_dep, &bound_modules); + } + } + for (Outer_join_module *outer_join_dep= table_dep->outer_join_dep; + outer_join_dep; outer_join_dep= outer_join_dep->parent) + { + //if (!(outer_join_dep->missing_tables &= ~table_dep->table->map)) + if (outer_join_dep->unknown_args && + !--outer_join_dep->unknown_args) + { + /* Mark as bound and add to the list */ + outer_join_dep->next= bound_modules; + bound_modules= outer_join_dep; + } + } + break; + } + default: DBUG_ASSERT(0); } - bound_dep= next; } } - DBUG_VOID_RETURN; } @@ -1232,7 +1253,7 @@ } - +#if 0 #ifndef DBUG_OFF static void dbug_print_deps(Table_elimination *te) @@ -1243,7 +1264,7 @@ fprintf(DBUG_FILE,"deps {\n"); /* Start with printing equalities */ - for (Equality_dep *eq_dep= te->equality_deps; + for (Equality_module *eq_dep= te->equality_deps; eq_dep != te->equality_deps + te->n_equality_deps; eq_dep++) { char buf[128]; @@ -1261,13 +1282,13 @@ /* Then tables and their fields */ for (uint i=0; i < MAX_TABLES; i++) { - Table_dep *table_dep; + Table_value *table_dep; if ((table_dep= te->table_deps[i])) { /* Print table */ fprintf(DBUG_FILE, " table %s\n", table_dep->table->alias); /* Print fields */ - for (Field_dep *field_dep= table_dep->fields; field_dep; + for (Field_value *field_dep= table_dep->fields; field_dep; field_dep= field_dep->next_table_field) { fprintf(DBUG_FILE, " field %s.%s ->", table_dep->table->alias, @@ -1288,7 +1309,7 @@ } #endif - +#endif /** @} (end of group Table_Elimination) */

1 0

[Maria-developers] Updated (by Guest): improving mysqlbinlog output and doing rename (39)
by worklog-noreply＠askmonty.org 14 Aug '09

14 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: improving mysqlbinlog output and doing rename CREATION DATE..: Sun, 09 Aug 2009, 12:24 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Client-RawIdeaBin TASK ID........: 39 (http://askmonty.org/worklog/?tid=39) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 17 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Fri, 14 Aug 2009, 15:52)=-=- Title modified. --- /tmp/wklog.39.old.11123 2009-08-14 15:52:29.000000000 +0300 +++ /tmp/wklog.39.new.11123 2009-08-14 15:52:29.000000000 +0300 @@ -1 +1 @@ -Replication tasks +improving mysqlbinlog output and doing rename -=-=(Guest - Mon, 10 Aug 2009, 16:32)=-=- Adding 1 hour for Monty's initial work on starting the architecture review. Worked 1 hour and estimate 0 hours remain (original estimate increased by 1 hour). -=-=(Psergey - Mon, 10 Aug 2009, 15:59)=-=- Re-searched and added subtasks. Worked 16 hours and estimate 0 hours remain (original estimate increased by 16 hours). -=-=(Psergey - Mon, 10 Aug 2009, 15:31)=-=- Dependency created: 39 now depends on 41 -=-=(Guest - Mon, 10 Aug 2009, 14:52)=-=- Dependency created: 39 now depends on 40 -=-=(Psergey - Sun, 09 Aug 2009, 12:27)=-=- Dependency created: 39 now depends on 36 -=-=(Psergey - Sun, 09 Aug 2009, 12:24)=-=- Dependency created: 39 now depends on 38 -=-=(Psergey - Sun, 09 Aug 2009, 12:24)=-=- Dependency created: 39 now depends on 37 DESCRIPTION: A combine task for all replication tasks. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Guest): improving mysqlbinlog output and doing rename (39)
by worklog-noreply＠askmonty.org 14 Aug '09

14 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: improving mysqlbinlog output and doing rename CREATION DATE..: Sun, 09 Aug 2009, 12:24 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Client-RawIdeaBin TASK ID........: 39 (http://askmonty.org/worklog/?tid=39) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 17 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Guest - Fri, 14 Aug 2009, 15:52)=-=- Title modified. --- /tmp/wklog.39.old.11123 2009-08-14 15:52:29.000000000 +0300 +++ /tmp/wklog.39.new.11123 2009-08-14 15:52:29.000000000 +0300 @@ -1 +1 @@ -Replication tasks +improving mysqlbinlog output and doing rename -=-=(Guest - Mon, 10 Aug 2009, 16:32)=-=- Adding 1 hour for Monty's initial work on starting the architecture review. Worked 1 hour and estimate 0 hours remain (original estimate increased by 1 hour). -=-=(Psergey - Mon, 10 Aug 2009, 15:59)=-=- Re-searched and added subtasks. Worked 16 hours and estimate 0 hours remain (original estimate increased by 16 hours). -=-=(Psergey - Mon, 10 Aug 2009, 15:31)=-=- Dependency created: 39 now depends on 41 -=-=(Guest - Mon, 10 Aug 2009, 14:52)=-=- Dependency created: 39 now depends on 40 -=-=(Psergey - Sun, 09 Aug 2009, 12:27)=-=- Dependency created: 39 now depends on 36 -=-=(Psergey - Sun, 09 Aug 2009, 12:24)=-=- Dependency created: 39 now depends on 38 -=-=(Psergey - Sun, 09 Aug 2009, 12:24)=-=- Dependency created: 39 now depends on 37 DESCRIPTION: A combine task for all replication tasks. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Knielsen): Add a mysqlbinlog option to filter updates to certain tables (40)
by worklog-noreply＠askmonty.org 14 Aug '09

14 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add a mysqlbinlog option to filter updates to certain tables CREATION DATE..: Mon, 10 Aug 2009, 13:25 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: Psergey CATEGORY.......: Server-RawIdeaBin TASK ID........: 40 (http://askmonty.org/worklog/?tid=40) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Fri, 14 Aug 2009, 15:47)=-=- High-Level Specification modified. --- /tmp/wklog.40.old.10896 2009-08-14 15:47:39.000000000 +0300 +++ /tmp/wklog.40.new.10896 2009-08-14 15:47:39.000000000 +0300 @@ -72,3 +72,21 @@ /* !mysqlbinlog: updates t1,db3.t2 */ UPDATE t1 LEFT JOIN ... and further processing in mysqlbinlog will be trivial. + +2.4 Implement server functionality to ignore certain tables +----------------------------------------------------------- + +We could add a general facility in the server to ignore certain tables: + + SET SESSION ignored_tables = "db1.t1,db2.t2"; + +This would work similar to --replicate-ignore-table, but in a general way not +restricted to the slave SQL thread. + +It would then be trivial for mysqlbinlog to add such statements at the start +of the output, or probably the user could just do it manually with no need for +additional options for mysqlbinlog. + +It might be useful to integrate this with the code that already handles +--replicate-ignore-db and similar slave options. + -=-=(Psergey - Mon, 10 Aug 2009, 15:41)=-=- High-Level Specification modified. --- /tmp/wklog.40.old.12989 2009-08-10 15:41:23.000000000 +0300 +++ /tmp/wklog.40.new.12989 2009-08-10 15:41:23.000000000 +0300 @@ -1,6 +1,7 @@ - 1. Context ---------- +(See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global +overview) At the moment, the server has these replication slave options: --replicate-do-table=db.tbl -=-=(Guest - Mon, 10 Aug 2009, 14:52)=-=- Dependency created: 39 now depends on 40 -=-=(Guest - Mon, 10 Aug 2009, 14:51)=-=- High Level Description modified. --- /tmp/wklog.40.old.16985 2009-08-10 14:51:59.000000000 +0300 +++ /tmp/wklog.40.new.16985 2009-08-10 14:51:59.000000000 +0300 @@ -1,3 +1,4 @@ Replication slave can be set to filter updates to certain tables with ---replicate-[wild-]{do,ignore}-table options. This task is about adding similar -functionality to mysqlbinlog. +--replicate-[wild-]{do,ignore}-table options. + +This task is about adding similar functionality to mysqlbinlog. -=-=(Guest - Mon, 10 Aug 2009, 14:51)=-=- High-Level Specification modified. --- /tmp/wklog.40.old.16949 2009-08-10 14:51:33.000000000 +0300 +++ /tmp/wklog.40.new.16949 2009-08-10 14:51:33.000000000 +0300 @@ -1 +1,73 @@ +1. Context +---------- +At the moment, the server has these replication slave options: + + --replicate-do-table=db.tbl + --replicate-ignore-table=db.tbl + --replicate-wild-do-table=pattern.pattern + --replicate-wild-ignore-table=pattern.pattern + +They affect both RBR and SBR events. SBR events are checked after the +statement has been parsed, the server iterates over list of used tables and +checks them againist --replicate instructions. + +What is interesting is that this scheme still allows to update the ignored +table through a VIEW. + +2. Table filtering in mysqlbinlog +--------------------------------- + +Per-table filtering of RBR events is easy (as it is relatively easy to extract +the name of the table that the event applies to). + +Per-table filtering of SBR events is hard, as generally it is not apparent +which tables the statement refers to. + +This opens possible options: + +2.1 Put the parser into mysqlbinlog +----------------------------------- +Once we have a full parser in mysqlbinlog, we'll be able to check which tables +are used by a statement, and will allow to show behaviour identical to those +that one obtains when using --replicate-* slave options. + +(It is not clear how much effort is needed to put the parser into mysqlbinlog. +Any guesses?) + + +2.2 Use dumb regexp match +------------------------- +Use a really dumb approach. A query is considered to be modifying table X if +it matches an expression + +CREATE TABLE $tablename +DROP $tablename +UPDATE ...$tablename ... SET // here '...' can't contain the word 'SET' +DELETE ...$tablename ... WHERE // same as above +ALTER TABLE $tablename +.. etc (go get from the grammar) .. + +The advantage over doing the same in awk is that mysqlbinlog will also process +RBR statements, and together with that will provide a working solution for +those who are careful with their table names not mixing with string constants +and such. + +(TODO: string constants are of particular concern as they come from +[potentially hostile] users, unlike e.g. table aliases which come from +[not hostile] developers. Remove also all string constants before attempting +to do match?) + +2.3 Have the master put annotations +----------------------------------- +We could add a master option so that it injects into query a mark that tells +which tables the query will affect, e.g. for the query + + UPDATE t1 LEFT JOIN db3.t2 ON ... WHERE ... + + +the binlog will have + + /* !mysqlbinlog: updates t1,db3.t2 */ UPDATE t1 LEFT JOIN ... + +and further processing in mysqlbinlog will be trivial. DESCRIPTION: Replication slave can be set to filter updates to certain tables with --replicate-[wild-]{do,ignore}-table options. This task is about adding similar functionality to mysqlbinlog. HIGH-LEVEL SPECIFICATION: 1. Context ---------- (See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global overview) At the moment, the server has these replication slave options: --replicate-do-table=db.tbl --replicate-ignore-table=db.tbl --replicate-wild-do-table=pattern.pattern --replicate-wild-ignore-table=pattern.pattern They affect both RBR and SBR events. SBR events are checked after the statement has been parsed, the server iterates over list of used tables and checks them againist --replicate instructions. What is interesting is that this scheme still allows to update the ignored table through a VIEW. 2. Table filtering in mysqlbinlog --------------------------------- Per-table filtering of RBR events is easy (as it is relatively easy to extract the name of the table that the event applies to). Per-table filtering of SBR events is hard, as generally it is not apparent which tables the statement refers to. This opens possible options: 2.1 Put the parser into mysqlbinlog ----------------------------------- Once we have a full parser in mysqlbinlog, we'll be able to check which tables are used by a statement, and will allow to show behaviour identical to those that one obtains when using --replicate-* slave options. (It is not clear how much effort is needed to put the parser into mysqlbinlog. Any guesses?) 2.2 Use dumb regexp match ------------------------- Use a really dumb approach. A query is considered to be modifying table X if it matches an expression CREATE TABLE $tablename DROP $tablename UPDATE ...$tablename ... SET // here '...' can't contain the word 'SET' DELETE ...$tablename ... WHERE // same as above ALTER TABLE $tablename .. etc (go get from the grammar) .. The advantage over doing the same in awk is that mysqlbinlog will also process RBR statements, and together with that will provide a working solution for those who are careful with their table names not mixing with string constants and such. (TODO: string constants are of particular concern as they come from [potentially hostile] users, unlike e.g. table aliases which come from [not hostile] developers. Remove also all string constants before attempting to do match?) 2.3 Have the master put annotations ----------------------------------- We could add a master option so that it injects into query a mark that tells which tables the query will affect, e.g. for the query UPDATE t1 LEFT JOIN db3.t2 ON ... WHERE ... the binlog will have /* !mysqlbinlog: updates t1,db3.t2 */ UPDATE t1 LEFT JOIN ... and further processing in mysqlbinlog will be trivial. 2.4 Implement server functionality to ignore certain tables ----------------------------------------------------------- We could add a general facility in the server to ignore certain tables: SET SESSION ignored_tables = "db1.t1,db2.t2"; This would work similar to --replicate-ignore-table, but in a general way not restricted to the slave SQL thread. It would then be trivial for mysqlbinlog to add such statements at the start of the output, or probably the user could just do it manually with no need for additional options for mysqlbinlog. It might be useful to integrate this with the code that already handles --replicate-ignore-db and similar slave options. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Knielsen): Add a mysqlbinlog option to filter updates to certain tables (40)
by worklog-noreply＠askmonty.org 14 Aug '09

14 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add a mysqlbinlog option to filter updates to certain tables CREATION DATE..: Mon, 10 Aug 2009, 13:25 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: Psergey CATEGORY.......: Server-RawIdeaBin TASK ID........: 40 (http://askmonty.org/worklog/?tid=40) VERSION........: Server-9.x STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Fri, 14 Aug 2009, 15:47)=-=- High-Level Specification modified. --- /tmp/wklog.40.old.10896 2009-08-14 15:47:39.000000000 +0300 +++ /tmp/wklog.40.new.10896 2009-08-14 15:47:39.000000000 +0300 @@ -72,3 +72,21 @@ /* !mysqlbinlog: updates t1,db3.t2 */ UPDATE t1 LEFT JOIN ... and further processing in mysqlbinlog will be trivial. + +2.4 Implement server functionality to ignore certain tables +----------------------------------------------------------- + +We could add a general facility in the server to ignore certain tables: + + SET SESSION ignored_tables = "db1.t1,db2.t2"; + +This would work similar to --replicate-ignore-table, but in a general way not +restricted to the slave SQL thread. + +It would then be trivial for mysqlbinlog to add such statements at the start +of the output, or probably the user could just do it manually with no need for +additional options for mysqlbinlog. + +It might be useful to integrate this with the code that already handles +--replicate-ignore-db and similar slave options. + -=-=(Psergey - Mon, 10 Aug 2009, 15:41)=-=- High-Level Specification modified. --- /tmp/wklog.40.old.12989 2009-08-10 15:41:23.000000000 +0300 +++ /tmp/wklog.40.new.12989 2009-08-10 15:41:23.000000000 +0300 @@ -1,6 +1,7 @@ - 1. Context ---------- +(See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global +overview) At the moment, the server has these replication slave options: --replicate-do-table=db.tbl -=-=(Guest - Mon, 10 Aug 2009, 14:52)=-=- Dependency created: 39 now depends on 40 -=-=(Guest - Mon, 10 Aug 2009, 14:51)=-=- High Level Description modified. --- /tmp/wklog.40.old.16985 2009-08-10 14:51:59.000000000 +0300 +++ /tmp/wklog.40.new.16985 2009-08-10 14:51:59.000000000 +0300 @@ -1,3 +1,4 @@ Replication slave can be set to filter updates to certain tables with ---replicate-[wild-]{do,ignore}-table options. This task is about adding similar -functionality to mysqlbinlog. +--replicate-[wild-]{do,ignore}-table options. + +This task is about adding similar functionality to mysqlbinlog. -=-=(Guest - Mon, 10 Aug 2009, 14:51)=-=- High-Level Specification modified. --- /tmp/wklog.40.old.16949 2009-08-10 14:51:33.000000000 +0300 +++ /tmp/wklog.40.new.16949 2009-08-10 14:51:33.000000000 +0300 @@ -1 +1,73 @@ +1. Context +---------- +At the moment, the server has these replication slave options: + + --replicate-do-table=db.tbl + --replicate-ignore-table=db.tbl + --replicate-wild-do-table=pattern.pattern + --replicate-wild-ignore-table=pattern.pattern + +They affect both RBR and SBR events. SBR events are checked after the +statement has been parsed, the server iterates over list of used tables and +checks them againist --replicate instructions. + +What is interesting is that this scheme still allows to update the ignored +table through a VIEW. + +2. Table filtering in mysqlbinlog +--------------------------------- + +Per-table filtering of RBR events is easy (as it is relatively easy to extract +the name of the table that the event applies to). + +Per-table filtering of SBR events is hard, as generally it is not apparent +which tables the statement refers to. + +This opens possible options: + +2.1 Put the parser into mysqlbinlog +----------------------------------- +Once we have a full parser in mysqlbinlog, we'll be able to check which tables +are used by a statement, and will allow to show behaviour identical to those +that one obtains when using --replicate-* slave options. + +(It is not clear how much effort is needed to put the parser into mysqlbinlog. +Any guesses?) + + +2.2 Use dumb regexp match +------------------------- +Use a really dumb approach. A query is considered to be modifying table X if +it matches an expression + +CREATE TABLE $tablename +DROP $tablename +UPDATE ...$tablename ... SET // here '...' can't contain the word 'SET' +DELETE ...$tablename ... WHERE // same as above +ALTER TABLE $tablename +.. etc (go get from the grammar) .. + +The advantage over doing the same in awk is that mysqlbinlog will also process +RBR statements, and together with that will provide a working solution for +those who are careful with their table names not mixing with string constants +and such. + +(TODO: string constants are of particular concern as they come from +[potentially hostile] users, unlike e.g. table aliases which come from +[not hostile] developers. Remove also all string constants before attempting +to do match?) + +2.3 Have the master put annotations +----------------------------------- +We could add a master option so that it injects into query a mark that tells +which tables the query will affect, e.g. for the query + + UPDATE t1 LEFT JOIN db3.t2 ON ... WHERE ... + + +the binlog will have + + /* !mysqlbinlog: updates t1,db3.t2 */ UPDATE t1 LEFT JOIN ... + +and further processing in mysqlbinlog will be trivial. DESCRIPTION: Replication slave can be set to filter updates to certain tables with --replicate-[wild-]{do,ignore}-table options. This task is about adding similar functionality to mysqlbinlog. HIGH-LEVEL SPECIFICATION: 1. Context ---------- (See http://askmonty.org/wiki/index.php/Scratch/ReplicationOptions for global overview) At the moment, the server has these replication slave options: --replicate-do-table=db.tbl --replicate-ignore-table=db.tbl --replicate-wild-do-table=pattern.pattern --replicate-wild-ignore-table=pattern.pattern They affect both RBR and SBR events. SBR events are checked after the statement has been parsed, the server iterates over list of used tables and checks them againist --replicate instructions. What is interesting is that this scheme still allows to update the ignored table through a VIEW. 2. Table filtering in mysqlbinlog --------------------------------- Per-table filtering of RBR events is easy (as it is relatively easy to extract the name of the table that the event applies to). Per-table filtering of SBR events is hard, as generally it is not apparent which tables the statement refers to. This opens possible options: 2.1 Put the parser into mysqlbinlog ----------------------------------- Once we have a full parser in mysqlbinlog, we'll be able to check which tables are used by a statement, and will allow to show behaviour identical to those that one obtains when using --replicate-* slave options. (It is not clear how much effort is needed to put the parser into mysqlbinlog. Any guesses?) 2.2 Use dumb regexp match ------------------------- Use a really dumb approach. A query is considered to be modifying table X if it matches an expression CREATE TABLE $tablename DROP $tablename UPDATE ...$tablename ... SET // here '...' can't contain the word 'SET' DELETE ...$tablename ... WHERE // same as above ALTER TABLE $tablename .. etc (go get from the grammar) .. The advantage over doing the same in awk is that mysqlbinlog will also process RBR statements, and together with that will provide a working solution for those who are careful with their table names not mixing with string constants and such. (TODO: string constants are of particular concern as they come from [potentially hostile] users, unlike e.g. table aliases which come from [not hostile] developers. Remove also all string constants before attempting to do match?) 2.3 Have the master put annotations ----------------------------------- We could add a master option so that it injects into query a mark that tells which tables the query will affect, e.g. for the query UPDATE t1 LEFT JOIN db3.t2 ON ... WHERE ... the binlog will have /* !mysqlbinlog: updates t1,db3.t2 */ UPDATE t1 LEFT JOIN ... and further processing in mysqlbinlog will be trivial. 2.4 Implement server functionality to ignore certain tables ----------------------------------------------------------- We could add a general facility in the server to ignore certain tables: SET SESSION ignored_tables = "db1.t1,db2.t2"; This would work similar to --replicate-ignore-table, but in a general way not restricted to the slave SQL thread. It would then be trivial for mysqlbinlog to add such statements at the start of the output, or probably the user could just do it manually with no need for additional options for mysqlbinlog. It might be useful to integrate this with the code that already handles --replicate-ignore-db and similar slave options. ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0

[Maria-developers] Updated (by Knielsen): Add a mysqlbinlog option to filter certain kinds of statements (41)
by worklog-noreply＠askmonty.org 14 Aug '09

14 Aug '09

----------------------------------------------------------------------- WORKLOG TASK -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- TASK...........: Add a mysqlbinlog option to filter certain kinds of statements CREATION DATE..: Mon, 10 Aug 2009, 15:30 SUPERVISOR.....: Monty IMPLEMENTOR....: COPIES TO......: CATEGORY.......: Client-BackLog TASK ID........: 41 (http://askmonty.org/worklog/?tid=41) VERSION........: Benchmarks-3.0 STATUS.........: Un-Assigned PRIORITY.......: 60 WORKED HOURS...: 0 ESTIMATE.......: 0 (hours remain) ORIG. ESTIMATE.: 0 PROGRESS NOTES: -=-=(Knielsen - Fri, 14 Aug 2009, 14:17)=-=- High-Level Specification modified. --- /tmp/wklog.41.old.6963 2009-08-14 14:17:32.000000000 +0300 +++ /tmp/wklog.41.new.6963 2009-08-14 14:17:32.000000000 +0300 @@ -1,6 +1,11 @@ The implementation will depend on design choices made in WL#40: -- If we decide to parse the statement, SQL-verb filtering will be trivial -- If we decide not to parse the statement, we still can reliably distinguish the + +Option 1: + +If we decide to parse the statement, SQL-verb filtering will be trivial + +Option 2: +If we decide not to parse the statement, we still can reliably distinguish the statement by matching the first characters against a set of patterns. If we chose the second, we'll have to perform certain normalization before -=-=(Psergey - Mon, 10 Aug 2009, 15:47)=-=- High-Level Specification modified. --- /tmp/wklog.41.old.13282 2009-08-10 15:47:13.000000000 +0300 +++ /tmp/wklog.41.new.13282 2009-08-10 15:47:13.000000000 +0300 @@ -2,3 +2,10 @@ - If we decide to parse the statement, SQL-verb filtering will be trivial - If we decide not to parse the statement, we still can reliably distinguish the statement by matching the first characters against a set of patterns. + +If we chose the second, we'll have to perform certain normalization before +matching the patterns: + - Remove all comments from the command + - Remove all pre-space + - Compare the string case-insensitively + - etc -=-=(Psergey - Mon, 10 Aug 2009, 15:35)=-=- High-Level Specification modified. --- /tmp/wklog.41.old.12689 2009-08-10 15:35:04.000000000 +0300 +++ /tmp/wklog.41.new.12689 2009-08-10 15:35:04.000000000 +0300 @@ -1 +1,4 @@ - +The implementation will depend on design choices made in WL#40: +- If we decide to parse the statement, SQL-verb filtering will be trivial +- If we decide not to parse the statement, we still can reliably distinguish the +statement by matching the first characters against a set of patterns. -=-=(Psergey - Mon, 10 Aug 2009, 15:31)=-=- Dependency created: 39 now depends on 41 DESCRIPTION: Add a mysqlbinlog option to filter certain kinds of statements, i.e. (syntax subject to discussion): mysqlbinlog --exclude='alter table,drop table,alter database,...' HIGH-LEVEL SPECIFICATION: The implementation will depend on design choices made in WL#40: Option 1: If we decide to parse the statement, SQL-verb filtering will be trivial Option 2: If we decide not to parse the statement, we still can reliably distinguish the statement by matching the first characters against a set of patterns. If we chose the second, we'll have to perform certain normalization before matching the patterns: - Remove all comments from the command - Remove all pre-space - Compare the string case-insensitively - etc ESTIMATED WORK TIME ESTIMATED COMPLETION DATE ----------------------------------------------------------------------- WorkLog (v3.5.9)

1 0